Initial Redmine tooling and local plugin forks

plugins/redmine_event_outbox/README.md

@@ -0,0 +1,66 @@
+# Redmine Event Outbox
+
+Small Redmine 3.4-compatible plugin that records selected Redmine changes into a
+local database outbox table for external workers.
+
+## Events Captured
+
+- `issue.created`
+- `issue.updated`
+- `journal.created`
+- `contact.created` when `redmine_contacts` is installed
+- `contact.updated` when `redmine_contacts` is installed
+- `helpdesk_ticket.created` when `redmine_contacts_helpdesk` is installed
+- `helpdesk_ticket.updated` when `redmine_contacts_helpdesk` is installed
+- `journal_message.created` when `redmine_contacts_helpdesk` is installed
+- `journal_message.updated` when `redmine_contacts_helpdesk` is installed
+
+The plugin does not publish to Redis, RabbitMQ, webhooks, or external search
+services directly from Redmine request callbacks. It only writes local database
+rows.
+
+## Install
+
+Copy this directory to the Redmine plugins directory:
+
+```sh
+cp -a redmine_event_outbox /path/to/redmine/plugins/
+```
+
+Run the plugin migration:
+
+```sh
+cd /path/to/redmine
+RAILS_ENV=production bundle exec rake redmine:plugins:migrate NAME=redmine_event_outbox
+```
+
+Restart Redmine. For Passenger:
+
+```sh
+touch tmp/restart.txt
+```
+
+## Verify
+
+List the rake task:
+
+```sh
+RAILS_ENV=production bundle exec rake -T redmine_event_outbox
+```
+
+Dump pending events:
+
+```sh
+RAILS_ENV=production bundle exec rake redmine_event_outbox:dump LIMIT=20
+```
+
+Create or update a low-risk test issue, then run the dump task again. You should
+see JSON rows in `event_outbox_events`.
+
+## Notes
+
+- Payloads are JSON serialized into a `text` column for MySQL and Redmine 3.4
+  compatibility.
+- Outbox write failures are rescued and logged so normal Redmine saves are not
+  intentionally failed by this plugin.
+- Consumers should treat events as at-least-once and idempotent.

plugins/redmine_event_outbox/app/models/event_outbox_event.rb

@@ -0,0 +1,15 @@
+class EventOutboxEvent < ActiveRecord::Base
+  validates :event_type, :source_type, :source_id, :occurred_at, :payload, :presence => true
+
+  scope :pending, lambda { where(:processed_at => nil).order(:id) }
+
+  def payload_data
+    ActiveSupport::JSON.decode(payload)
+  rescue
+    {}
+  end
+
+  def payload_data=(data)
+    self.payload = ActiveSupport::JSON.encode(data)
+  end
+end

plugins/redmine_event_outbox/db/migrate/001_create_event_outbox_events.rb

@@ -0,0 +1,27 @@
+class CreateEventOutboxEvents < ActiveRecord::Migration
+  def change
+    create_table :event_outbox_events do |t|
+      t.string :event_type, :null => false
+      t.string :source_type, :null => false
+      t.integer :source_id, :null => false
+      t.integer :project_id
+      t.integer :issue_id
+      t.integer :journal_id
+      t.integer :user_id
+      t.datetime :occurred_at, :null => false
+      t.text :payload, :null => false
+      t.datetime :processed_at
+      t.integer :attempts, :null => false, :default => 0
+      t.text :last_error
+      t.datetime :locked_at
+      t.string :locked_by
+      t.timestamps
+    end
+
+    add_index :event_outbox_events, [:processed_at, :id]
+    add_index :event_outbox_events, :event_type
+    add_index :event_outbox_events, :issue_id
+    add_index :event_outbox_events, :project_id
+    add_index :event_outbox_events, :occurred_at
+  end
+end

plugins/redmine_event_outbox/init.rb

@@ -0,0 +1,11 @@
+require 'redmine'
+
+Redmine::Plugin.register :redmine_event_outbox do
+  name 'Redmine Event Outbox'
+  author 'Internal'
+  description 'Records Redmine changes into a local outbox table for external workers.'
+  version '0.0.1'
+  requires_redmine :version_or_higher => '3.4'
+end
+
+require_dependency 'redmine_event_outbox'

plugins/redmine_event_outbox/lib/redmine_event_outbox.rb

@@ -0,0 +1,36 @@
+require_dependency 'redmine_event_outbox/event'
+require_dependency 'redmine_event_outbox/hooks/issues_hook'
+
+ActionDispatch::Callbacks.to_prepare do
+  require_dependency 'redmine_event_outbox/patches/journal_patch'
+  Journal.send(:include, RedmineEventOutbox::Patches::JournalPatch) unless Journal.included_modules.include?(RedmineEventOutbox::Patches::JournalPatch)
+
+  if defined?(Contact)
+    require_dependency 'redmine_event_outbox/patches/contact_patch'
+    Contact.send(:include, RedmineEventOutbox::Patches::ContactPatch) unless Contact.included_modules.include?(RedmineEventOutbox::Patches::ContactPatch)
+  end
+
+  # Optional local integration with the installed RedmineUP helpdesk fork.
+  # The outbox plugin stays loadable without helpdesk, but captures first-class
+  # helpdesk identity when the plugin is present.
+  helpdesk_installed = begin
+    Redmine::Plugin.installed?(:redmine_contacts_helpdesk)
+  rescue
+    false
+  end
+
+  if helpdesk_installed
+    require_dependency 'helpdesk_ticket'
+    require_dependency 'journal_message'
+
+    if defined?(HelpdeskTicket)
+      require_dependency 'redmine_event_outbox/patches/helpdesk_ticket_patch'
+      HelpdeskTicket.send(:include, RedmineEventOutbox::Patches::HelpdeskTicketPatch) unless HelpdeskTicket.included_modules.include?(RedmineEventOutbox::Patches::HelpdeskTicketPatch)
+    end
+
+    if defined?(JournalMessage)
+      require_dependency 'redmine_event_outbox/patches/journal_message_patch'
+      JournalMessage.send(:include, RedmineEventOutbox::Patches::JournalMessagePatch) unless JournalMessage.included_modules.include?(RedmineEventOutbox::Patches::JournalMessagePatch)
+    end
+  end
+end

plugins/redmine_event_outbox/lib/redmine_event_outbox/event.rb

@@ -0,0 +1,268 @@
+module RedmineEventOutbox
+  module Event
+    module_function
+
+    def record_issue_created(issue, actor)
+      record(
+        :event_type => 'issue.created',
+        :source => issue,
+        :project_id => issue.project_id,
+        :issue_id => issue.id,
+        :user_id => issue.author_id,
+        :payload => issue_payload(issue, actor, 'issue.created')
+      )
+    end
+
+    def record_issue_updated(issue, journal, actor)
+      record(
+        :event_type => 'issue.updated',
+        :source => issue,
+        :project_id => issue.project_id,
+        :issue_id => issue.id,
+        :journal_id => journal.try(:id),
+        :user_id => actor.try(:id),
+        :payload => issue_payload(issue, actor, 'issue.updated').merge(
+          :journal_id => journal.try(:id),
+          :changed_fields => journal_changed_fields(journal)
+        )
+      )
+    end
+
+    def record_journal_created(journal, actor)
+      return unless journal && journal.journalized_type == 'Issue'
+
+      issue = journal.issue || journal.journalized
+      return unless issue
+
+      record(
+        :event_type => 'journal.created',
+        :source => journal,
+        :project_id => issue.project_id,
+        :issue_id => issue.id,
+        :journal_id => journal.id,
+        :user_id => journal.user_id,
+        :payload => journal_payload(journal, issue, actor)
+      )
+    end
+
+    def record_contact_created(contact, actor)
+      record_contact_event('contact.created', contact, actor)
+    end
+
+    def record_contact_updated(contact, actor)
+      record_contact_event('contact.updated', contact, actor)
+    end
+
+    def record_helpdesk_ticket_created(helpdesk_ticket, actor)
+      record_helpdesk_ticket_event('helpdesk_ticket.created', helpdesk_ticket, actor)
+    end
+
+    def record_helpdesk_ticket_updated(helpdesk_ticket, actor)
+      record_helpdesk_ticket_event('helpdesk_ticket.updated', helpdesk_ticket, actor)
+    end
+
+    def record_journal_message_created(journal_message, actor)
+      record_journal_message_event('journal_message.created', journal_message, actor)
+    end
+
+    def record_journal_message_updated(journal_message, actor)
+      record_journal_message_event('journal_message.updated', journal_message, actor)
+    end
+
+    def record_contact_event(event_type, contact, actor)
+      record(
+        :event_type => event_type,
+        :source => contact,
+        :project_id => primary_project_id(contact),
+        :user_id => actor.try(:id),
+        :payload => contact_payload(contact, actor, event_type)
+      )
+    end
+
+    def record_helpdesk_ticket_event(event_type, helpdesk_ticket, actor)
+      issue = helpdesk_ticket.issue
+
+      record(
+        :event_type => event_type,
+        :source => helpdesk_ticket,
+        :project_id => issue.try(:project_id),
+        :issue_id => helpdesk_ticket.issue_id,
+        :user_id => actor.try(:id),
+        :payload => helpdesk_ticket_payload(helpdesk_ticket, issue, actor, event_type)
+      )
+    end
+
+    def record_journal_message_event(event_type, journal_message, actor)
+      journal = journal_message.journal
+      issue = journal.try(:issue) || journal.try(:journalized)
+
+      record(
+        :event_type => event_type,
+        :source => journal_message,
+        :project_id => issue.try(:project_id),
+        :issue_id => issue.try(:id),
+        :journal_id => journal_message.journal_id,
+        :user_id => journal.try(:user_id) || actor.try(:id),
+        :payload => journal_message_payload(journal_message, journal, issue, actor, event_type)
+      )
+    end
+
+    def record(attributes)
+      source = attributes.fetch(:source)
+      payload = attributes.fetch(:payload)
+
+      event = EventOutboxEvent.new(
+        :event_type => attributes.fetch(:event_type),
+        :source_type => source.class.name,
+        :source_id => source.id,
+        :project_id => attributes[:project_id],
+        :issue_id => attributes[:issue_id],
+        :journal_id => attributes[:journal_id],
+        :user_id => attributes[:user_id],
+        :occurred_at => Time.now
+      )
+      event.payload_data = payload
+      event.save!
+    rescue => e
+      Rails.logger.error(
+        "RedmineEventOutbox: failed to record #{attributes[:event_type]} " \
+        "for #{source.class.name}##{source.try(:id)}: #{e.class}: #{e.message}"
+      )
+      nil
+    end
+
+    def issue_payload(issue, actor, event_type)
+      {
+        :event_type => event_type,
+        :issue_id => issue.id,
+        :project_id => issue.project_id,
+        :tracker_id => issue.tracker_id,
+        :status_id => issue.status_id,
+        :priority_id => issue.priority_id,
+        :author_id => issue.author_id,
+        :author_name => principal_name(issue.author),
+        :assigned_to_id => issue.assigned_to_id,
+        :assigned_to_name => principal_name(issue.assigned_to),
+        :actor_id => actor.try(:id),
+        :actor_name => principal_name(actor),
+        :subject => issue.subject,
+        :created_on => iso8601(issue.created_on),
+        :updated_on => iso8601(issue.updated_on)
+      }
+    end
+
+    def journal_payload(journal, issue, actor)
+      {
+        :event_type => 'journal.created',
+        :journal_id => journal.id,
+        :issue_id => issue.id,
+        :project_id => issue.project_id,
+        :user_id => journal.user_id,
+        :user_name => principal_name(journal.user),
+        :actor_id => actor.try(:id),
+        :actor_name => principal_name(actor),
+        :subject => issue.subject,
+        :private_notes => journal.private_notes?,
+        :has_notes => journal.notes.present?,
+        :changed_fields => journal_changed_fields(journal),
+        :created_on => iso8601(journal.created_on)
+      }
+    end
+
+    def contact_payload(contact, actor, event_type)
+      {
+        :event_type => event_type,
+        :contact_id => contact.id,
+        :project_ids => contact_project_ids(contact),
+        :is_company => contact.is_company,
+        :name => contact_name(contact),
+        :company => contact.try(:company),
+        :author_id => contact.try(:author_id),
+        :author_name => principal_name(contact.try(:author)),
+        :assigned_to_id => contact.try(:assigned_to_id),
+        :assigned_to_name => principal_name(contact.try(:assigned_to)),
+        :actor_id => actor.try(:id),
+        :actor_name => principal_name(actor),
+        :created_on => iso8601(contact.try(:created_on)),
+        :updated_on => iso8601(contact.try(:updated_on))
+      }
+    end
+
+    def helpdesk_ticket_payload(helpdesk_ticket, issue, actor, event_type)
+      {
+        :event_type => event_type,
+        :helpdesk_ticket_id => helpdesk_ticket.id,
+        :issue_id => helpdesk_ticket.issue_id,
+        :project_id => issue.try(:project_id),
+        :contact_id => helpdesk_ticket.contact_id,
+        :message_id => helpdesk_ticket.message_id,
+        :is_incoming => helpdesk_ticket.is_incoming?,
+        :source => helpdesk_ticket.source,
+        :from_address => helpdesk_ticket.from_address,
+        :to_address => helpdesk_ticket.to_address,
+        :cc_address => helpdesk_ticket.cc_address,
+        :actor_id => actor.try(:id),
+        :actor_name => principal_name(actor),
+        :subject => issue.try(:subject),
+        :ticket_date => iso8601(helpdesk_ticket.try(:ticket_date)),
+        :updated_on => iso8601(helpdesk_ticket.try(:updated_on))
+      }
+    end
+
+    def journal_message_payload(journal_message, journal, issue, actor, event_type)
+      # Keep event rows useful for routing/index invalidation without copying
+      # email bodies, private notes, attachments, or BCC addresses into outbox.
+      {
+        :event_type => event_type,
+        :journal_message_id => journal_message.id,
+        :journal_id => journal_message.journal_id,
+        :issue_id => issue.try(:id),
+        :project_id => issue.try(:project_id),
+        :contact_id => journal_message.contact_id,
+        :message_id => journal_message.message_id,
+        :is_incoming => journal_message.is_incoming?,
+        :source => journal_message.source,
+        :from_address => journal_message.from_address,
+        :to_address => journal_message.to_address,
+        :cc_address => journal_message.cc_address,
+        :has_bcc_address => journal_message.bcc_address.present?,
+        :user_id => journal.try(:user_id),
+        :user_name => principal_name(journal.try(:user)),
+        :actor_id => actor.try(:id),
+        :actor_name => principal_name(actor),
+        :subject => issue.try(:subject),
+        :private_notes => journal.try(:private_notes?),
+        :has_notes => journal.try(:notes).present?,
+        :message_date => iso8601(journal_message.try(:message_date))
+      }
+    end
+
+    def journal_changed_fields(journal)
+      return [] unless journal && journal.respond_to?(:details)
+      journal.details.map(&:prop_key).compact.uniq
+    end
+
+    def contact_project_ids(contact)
+      return [] unless contact.respond_to?(:projects)
+      contact.projects.map(&:id).compact
+    rescue
+      []
+    end
+
+    def primary_project_id(contact)
+      contact_project_ids(contact).first
+    end
+
+    def contact_name(contact)
+      contact.try(:name).presence || [contact.try(:first_name), contact.try(:last_name)].compact.join(' ').presence
+    end
+
+    def principal_name(principal)
+      principal.try(:name).presence if principal
+    end
+
+    def iso8601(value)
+      value.try(:utc).try(:iso8601)
+    end
+  end
+end

plugins/redmine_event_outbox/lib/redmine_event_outbox/hooks/issues_hook.rb

@@ -0,0 +1,24 @@
+module RedmineEventOutbox
+  module Hooks
+    class IssuesHook < Redmine::Hook::ViewListener
+      def controller_issues_new_after_save(context = {})
+        return unless context[:issue]
+
+        RedmineEventOutbox::Event.record_issue_created(
+          context[:issue],
+          User.current
+        )
+      end
+
+      def controller_issues_edit_after_save(context = {})
+        return unless context[:issue]
+
+        RedmineEventOutbox::Event.record_issue_updated(
+          context[:issue],
+          context[:journal],
+          User.current
+        )
+      end
+    end
+  end
+end

plugins/redmine_event_outbox/lib/redmine_event_outbox/patches/contact_patch.rb

@@ -0,0 +1,22 @@
+module RedmineEventOutbox
+  module Patches
+    module ContactPatch
+      def self.included(base)
+        base.class_eval do
+          after_commit :record_event_outbox_contact_created, :on => :create
+          after_commit :record_event_outbox_contact_updated, :on => :update
+        end
+      end
+
+      private
+
+      def record_event_outbox_contact_created
+        RedmineEventOutbox::Event.record_contact_created(self, User.current)
+      end
+
+      def record_event_outbox_contact_updated
+        RedmineEventOutbox::Event.record_contact_updated(self, User.current)
+      end
+    end
+  end
+end

plugins/redmine_event_outbox/lib/redmine_event_outbox/patches/helpdesk_ticket_patch.rb

@@ -0,0 +1,22 @@
+module RedmineEventOutbox
+  module Patches
+    module HelpdeskTicketPatch
+      def self.included(base)
+        base.class_eval do
+          # Local fork hook: helpdesk ticket rows carry the real customer/email
+          # identity for many tickets whose Redmine issue author is Anonymous.
+          after_commit :record_event_outbox_helpdesk_ticket_created, :on => :create
+          after_commit :record_event_outbox_helpdesk_ticket_updated, :on => :update
+        end
+      end
+
+      def record_event_outbox_helpdesk_ticket_created
+        RedmineEventOutbox::Event.record_helpdesk_ticket_created(self, User.current)
+      end
+
+      def record_event_outbox_helpdesk_ticket_updated
+        RedmineEventOutbox::Event.record_helpdesk_ticket_updated(self, User.current)
+      end
+    end
+  end
+end

plugins/redmine_event_outbox/lib/redmine_event_outbox/patches/journal_message_patch.rb

@@ -0,0 +1,22 @@
+module RedmineEventOutbox
+  module Patches
+    module JournalMessagePatch
+      def self.included(base)
+        base.class_eval do
+          # Local fork hook: JournalMessage is the per-email metadata layer for
+          # helpdesk conversations, so indexers should react to it directly.
+          after_commit :record_event_outbox_journal_message_created, :on => :create
+          after_commit :record_event_outbox_journal_message_updated, :on => :update
+        end
+      end
+
+      def record_event_outbox_journal_message_created
+        RedmineEventOutbox::Event.record_journal_message_created(self, User.current)
+      end
+
+      def record_event_outbox_journal_message_updated
+        RedmineEventOutbox::Event.record_journal_message_updated(self, User.current)
+      end
+    end
+  end
+end

plugins/redmine_event_outbox/lib/redmine_event_outbox/patches/journal_patch.rb

@@ -0,0 +1,17 @@
+module RedmineEventOutbox
+  module Patches
+    module JournalPatch
+      def self.included(base)
+        base.class_eval do
+          after_commit :record_event_outbox_journal_created, :on => :create
+        end
+      end
+
+      private
+
+      def record_event_outbox_journal_created
+        RedmineEventOutbox::Event.record_journal_created(self, User.current)
+      end
+    end
+  end
+end

plugins/redmine_event_outbox/lib/tasks/redmine_event_outbox.rake

@@ -0,0 +1,22 @@
+namespace :redmine_event_outbox do
+  desc 'Print pending Redmine event outbox rows as JSON.'
+  task :dump => :environment do
+    limit = ENV['LIMIT'].to_i
+    limit = 100 if limit <= 0
+
+    EventOutboxEvent.pending.limit(limit).each do |event|
+      puts ActiveSupport::JSON.encode(
+        :id => event.id,
+        :event_type => event.event_type,
+        :source_type => event.source_type,
+        :source_id => event.source_id,
+        :project_id => event.project_id,
+        :issue_id => event.issue_id,
+        :journal_id => event.journal_id,
+        :user_id => event.user_id,
+        :occurred_at => event.occurred_at.try(:utc).try(:iso8601),
+        :payload => event.payload_data
+      )
+    end
+  end
+end