init

lib/modules/channels.js

@@ -0,0 +1,114 @@
+const _ = require('lodash')
+const slug = require('slug')
+const XML = require('xmldoc')
+const Factory = require('../factory')
+
+//
+// Initialize the child logger for
+// the module
+//
+const log = require('../log').child({
+  module: 'channels'
+})
+
+module.exports = function(context) {
+  //
+  // Select all of the rooms
+  //
+  return context.jabber.fetch(
+    'SELECT room_jid, subject, config FROM dbo.tc_rooms'
+  )
+    //
+    // The convert them to channels and
+    // write them to the datafile
+    //
+    .then(function(results) {
+      log.info(`${results.recordset.length} records found`)
+      //
+      // Define a map to hold the channels
+      // info for downstream modules
+      //
+      var channels = {}
+      //
+      // Iterate over the record set and
+      // create a channel for each room
+      //
+      results.recordset.forEach(function(room) {
+        log.debug(room)
+        //
+        // Define the channel and add
+        // it to the context
+        //
+        var channel = channels[room.room_jid] = {
+          team: context.values.team.name,
+          name: slug(toName(room.room_jid)),
+          display_name: toDisplayName(room.room_jid),
+          header: toDescription(room.subject),
+          purpose: toDescription(room.subject),
+          type: toType(room.config)
+        }
+        //
+        // Write the channel data to the
+        // output
+        //
+        log.info(`... writing ${channel.name}`)
+        context.output.write(
+          Factory.channel(channel)
+        )
+      })
+      //
+      // Add the channel map to the context
+      //
+      context.values.channels = channels
+      //
+      // Resolve the promise
+      //
+      return context
+    })
+}
+
+//
+// Parses the name from a room jid
+//
+const toName = function (jid='') {
+  return jid.substr(0, jid.indexOf('@')).replace(/\\20/g, ' ')
+}
+
+//
+// Formats the display name
+//
+const toDisplayName = function (jid='') {
+  return _.startCase(_.toLower(toName(jid)))
+}
+
+//
+// Returns a description from the subject
+//
+const toDescription = function (subject='') {
+  return subject
+}
+
+//
+// Parses the channel type from the
+// config field
+//
+const toType = function (xml='<x/>') {
+  //
+  // Parse the config
+  //
+  var config = new XML.XmlDocument(xml)
+  //
+  // Extract the value. If it does not exist,
+  // we default to '1' - private
+  //
+  var value = _.get(
+    config.childWithAttribute('var', 'members-only'),
+    'firstChild.val',
+    '1'
+  )
+  //
+  // If members-only is '1', return private
+  // otherwise, return public
+  //
+  return value === '1' ? 'P' : 'O'
+}

lib/modules/directChannels.js

@@ -0,0 +1,89 @@
+const Factory = require('../factory')
+const transform = require('./transform')
+const Utils = require('./utils')
+
+//
+// Initialize the child logger for
+// the module
+//
+const log = require('../log').child({
+  module: 'directChannels'
+})
+
+module.exports = function(context) {
+  return new Promise(function(resolve /*, reject */) {
+    log.info('streaming records')
+    //
+    // Array to accumulate the channel
+    // member pairs
+    //
+    var channels = {}
+    //
+    // Query messages from Jabber and pipe
+    // through the post transform and
+    // then to the output. We use pipe to
+    // handle very large data sets using
+    // streams
+    //
+    context.jabber.pipe(
+      //
+      // Define the query
+      //
+      `SELECT DISTINCT to_jid, from_jid FROM dbo.jm
+        WHERE msg_type = 'c'
+          AND direction = 'I'
+          AND (body_string != '' or datalength(body_text) > 0)`,
+      //
+      // Define the tranform
+      //
+      transform(function(result, encoding, callback) {
+        try {
+          log.debug(result)
+          //
+          // Generate the members array for the
+          // direct channel
+          //
+          let members = Utils.members(
+            context.values.users,
+            result.to_jid,
+            result.from_jid
+          )
+          //
+          // If there at least two members
+          //
+          if(Utils.membersAreValid(members)) {
+            var key = `${members[0]}|${members[1]}`
+            //
+            // And we haven't processed this pair
+            // already
+            //
+            if(!channels[key]) {
+              channels[key] = members
+              log.info(`... writing ${members}`)
+              context.output.write(
+                Factory.directChannel({
+                  members
+                })
+              )
+            }
+          }
+        } catch (err) {
+          log.error(`... ignoring directChannel from: ${result.from_jid} to: ${result.to_jid} on error: ${err.message}.`)
+        }
+        //
+        // Invoke the call to mark that we are
+        // done with the chunk
+        //
+        return callback()
+      },
+      //
+      // Define the callback to be invoked
+      // on finish
+      //
+      function() {
+        log.info('... finished')
+        resolve(context)
+      })
+    )
+  })
+}

lib/modules/directPosts.js

@@ -0,0 +1,106 @@
+const Factory = require('../factory')
+const transform = require('./transform')
+const Utils = require('./utils')
+
+//
+// Initialize the child logger for
+// the module
+//
+const log = require('../log').child({
+  module: 'directPosts'
+})
+
+module.exports = function(context) {
+  return new Promise(function(resolve /*, reject */) {
+    log.info('streaming records')
+    //
+    // Keep track of the number of posts
+    // written for logging
+    //
+    var written = 0
+    //
+    // Query messages from Jabber and pipe
+    // through the post transform and
+    // then to the output. We use pipe to
+    // handle very large data sets using
+    // streams
+    //
+    context.jabber.pipe(
+      //
+      // Define the query
+      //
+      `SELECT to_jid, from_jid, sent_date, body_string, body_text FROM dbo.jm
+          WHERE msg_type = 'c'
+            AND direction = 'I'
+            AND (body_string != '' or datalength(body_text) > 0)`,
+      //
+      // Define the tranform
+      //
+      transform(function(message, encoding, callback) {
+        try {
+          log.debug(message)
+          //
+          // Generate the members array for the
+          // direct channel
+          //
+          let members = Utils.members(
+            context.values.users,
+            message.to_jid,
+            message.from_jid
+          )
+          //
+          // Ensure we have at least two channel
+          // members before we can write the message
+          //
+          if (Utils.membersAreValid(members)) {
+            //
+            // Base post props
+            //
+            var post = {
+              channel_members: members,
+              user: Utils.username(context.values.users, message.from_jid),
+              create_at: Utils.millis(message.sent_date)
+            }
+            //
+            // Process each chunk
+            //
+            Utils.body(message).forEach(function(chunk) {
+              context.output.write(
+                Factory.directPost(Object.assign({}, post, {
+                  message: chunk
+                }))
+              )
+              //
+              // Log progress periodically
+              //
+              written += 1
+              if(written % 1000 == 0) {
+                log.info(`... wrote ${written} posts`)
+              }
+            })
+          } else {
+            log.warn({
+              to_jid: message.to_jid,
+              from_jid: message.from_jid
+            }, '... skipping message with invalid channel members')
+          }
+        } catch (err) {
+          log.error(`... ignoring directPost from: ${message.from_jid} to: ${message.to_jid} on error: ${err.message}.`)
+        }
+        //
+        // Invoke the call to mark that we are
+        // done with the chunk
+        //
+        return callback()
+      },
+      //
+      // Define the callback to be invoked
+      // on finish
+      //
+      function() {
+        log.info(`... finished writing ${written} posts`)
+        resolve(context)
+      })
+    )
+  })
+}

lib/modules/end.js

@@ -0,0 +1,12 @@
+//
+// Initialize the child logger for
+// the module
+//
+const log = require('../log').child({
+  module: 'end'
+})
+
+module.exports = function(context) {
+  log.info('end')
+  context.output.end()
+}

lib/modules/index.js

@@ -0,0 +1,21 @@
+const start = require('./start')
+const version = require('./version')
+const team = require('./team')
+const channels = require('./channels')
+const users = require('./users')
+const posts = require('./posts')
+const directChannels = require('./directChannels')
+const directPosts = require('./directPosts')
+const end = require('./end')
+
+module.exports = {
+  start,
+  version,
+  team,
+  channels,
+  users,
+  posts,
+  directChannels,
+  directPosts,
+  end
+}

lib/modules/posts.js

@@ -0,0 +1,100 @@
+const Factory = require('../factory')
+const transform = require('./transform')
+const Utils = require('./utils')
+
+//
+// Initialize the child logger for
+// the module
+//
+const log = require('../log').child({
+  module: 'posts'
+})
+
+module.exports = function(context) {
+  return new Promise(function(resolve /*, reject */) {
+    log.info('streaming records')
+    //
+    // Keep track of the number of posts
+    // written for logging
+    //
+    var written = 0
+    //
+    // Query messages from Jabber and pipe
+    // through the post transform and
+    // then to the output. We use pipe to
+    // handle very large data sets using
+    // streams
+    //
+    context.jabber.pipe(
+      //
+      // Define the query
+      //
+      `SELECT
+            msg_id,
+            to_jid,
+            from_jid,
+            sent_date,
+            body_string,
+            body_text
+        FROM dbo.tc_msgarchive WHERE msg_type = 'g' AND (body_string != '' OR datalength(body_text) > 0) `,
+      //
+      // Define the tranform
+      //
+      transform(function(message, encoding, callback) {
+        try {
+          log.debug(message)
+          //
+          // Base post props
+          //
+          var post = {
+            team: context.values.team.name,
+            channel: Utils.channelName(
+              context.values.channels, message.to_jid
+            ),
+            user: Utils.username(
+              context.values.users, message.from_jid
+            ),
+            create_at: Utils.millis(message.sent_date)
+          }
+          //
+          // Process each chunk
+          //
+          Utils.body(message).forEach(function(chunk) {
+            //
+            // Write the post object to the
+            // output
+            //
+            context.output.write(
+              Factory.post(Object.assign({}, post, {
+                message: chunk
+              }))
+            )
+            //
+            // Log progress periodically
+            //
+            written += 1
+            if(written % 1000 == 0) {
+              log.info(`... wrote ${written} posts`)
+            }
+          })
+        }
+        catch(err) {
+          log.error(`... ignoring message id:${message.msg_id} on error: ${err.message}.`)
+        }
+        //
+        // Invoke the call to mark that we are
+        // done with the chunk
+        //
+        return callback()
+      },
+      //
+      // Define the callback to be invoked
+      // on finish
+      //
+      function() {
+        log.info(`... finished writing ${written} posts`)
+        resolve(context)
+      })
+    )
+  })
+}

lib/modules/start.js

@@ -0,0 +1,32 @@
+const datafile = require('../datafile')
+
+//
+// Initialize the child logger for
+// the module
+//
+const log = require('../log').child({
+  module: 'start'
+})
+
+module.exports = function(context) {
+  log.info('connecting to jabber')
+  //
+  // Connect to the jabber database
+  //
+  return context.jabber.connect(context.config)
+    .then(function() {
+      log.info(`creating file '${context.config.target.filename}'`)
+      //
+      // Create the datafile and add
+      // it to the context
+      //
+      context.output = datafile(
+        context.config.target.filename,
+        process.exit
+      )
+      //
+      // Resolve the promise
+      //
+      return context
+    })
+}

lib/modules/team.js

@@ -0,0 +1,29 @@
+const Factory = require('../factory')
+
+//
+// Initialize the child logger for
+// the module
+//
+const log = require('../log').child({
+  module: 'team'
+})
+
+module.exports = function(context) {
+  log.info(`writing team '${context.config.define.team.name}'`)
+  //
+  // Store the team object for use
+  // by other modules
+  //
+  context.values.team = context.config.define.team
+  //
+  // Write the team object to the
+  // output
+  //
+  context.output.write(Factory.team(
+    context.config.define.team
+  ))
+  //
+  // Return a resolved promise
+  //
+  return Promise.resolve(context)
+}

lib/modules/test/channels.js

@@ -0,0 +1,61 @@
+const expect = require('chai').expect
+const channels = require('../channels')
+const Fixtures = require('./fixtures')
+const context = require('./context')()
+
+describe('modules.channels', function() {
+
+  it('should process channel objects', function(done) {
+
+    context.values.team = context.config.define.team
+
+    context.jabber.fetch.returns(Promise.resolve({
+      recordset: Fixtures.channels
+    }))
+
+    channels(context)
+      .then(function (c) {
+        expect(c).to.equal(context)
+        expect(Object.keys(c.values.channels).length).equals(3)
+        expect(c.output.write.args[0][0]).to.deep.equal({
+          type: 'channel',
+          channel: {
+            team: 'test',
+            name: 'admin',
+            display_name: 'Admin',
+            header: 'Admin Room',
+            purpose: 'Admin Room',
+            type: 'O'
+          }
+        })
+        expect(c.output.write.args[1][0]).to.deep.equal({
+          type: 'channel',
+          channel: {
+            team: 'test',
+            name: 'test-room',
+            display_name: 'Test Room',
+            header: '',
+            purpose: '',
+            type: 'O'
+          }
+        })
+        expect(c.output.write.args[2][0]).to.deep.equal({
+          type: 'channel',
+          channel: {
+            team: 'test',
+            name: 'tonyd_20161206_1432',
+            display_name: 'Tonyd 20161206 1432',
+            header: '',
+            purpose: '',
+            type: 'O'
+          }
+        })
+        done()
+      })
+  })
+
+  afterEach(function() {
+    context.jabber.fetch.reset()
+    context.output.write.reset()
+  })
+})

lib/modules/test/context.js

@@ -0,0 +1,38 @@
+const { spy, stub } = require('sinon')
+
+module.exports = function() {
+  return {
+    config: {
+      source: {
+        uri: 'mssql://username:password@server:1433/jabber?encrypt=true'
+      },
+      target: {
+        filename: 'data.json'
+      },
+      define: {
+        team: {
+          name: 'test',
+          display_name: 'Test Team',
+          description: 'Our Test Team',
+          type: 'I',
+          allow_open_invite: false
+        },
+        user: {
+          auth_service: 'ldap',
+        }
+      }
+    },
+    jabber: {
+      connect: stub().returns(Promise.resolve()),
+      fetch: stub(),
+      pipe: stub()
+    },
+    values: {
+
+    },
+    output: {
+      write: spy(),
+      end: spy()
+    }
+  }
+}

lib/modules/test/directChannels.js

@@ -0,0 +1,70 @@
+const expect = require('chai').expect
+const directChannels = require('../directChannels')
+const Fixtures = require('./fixtures')
+const FakeDB = require('./fakedb')
+const sink = require('./sink')
+const context = require('./context')()
+
+describe('modules.directChannels', function() {
+
+  before(function() {
+    //
+    // Set up context values
+    //
+    context.values = {
+      users: {
+        'person.one@example.com': {
+          username: 'person.one'
+        },
+        'person.two@example.com': {
+          username: 'person.two'
+        },
+        'person.three@example.com': {
+          username: 'person.three'
+        }
+      }
+    }
+  })
+
+  beforeEach(function() {
+    //
+    // Stub the output stream
+    //
+    context.output = sink()
+  })
+
+  it('should process direct channel objects', function(done) {
+    //
+    // Set up the DB
+    //
+    context.jabber = new FakeDB(Fixtures.directChannels)
+    //
+    // Process the data
+    //
+    directChannels(context).then(function(c) {
+      expect(c).to.equal(context)
+      expect(c.output.write.args).to.deep.equal(
+        [
+          [
+            {
+              type: 'direct_channel',
+              direct_channel: {
+                members: ['person.one', 'person.two']
+              }
+            }
+          ],
+          [
+            {
+              type: 'direct_channel',
+              direct_channel: {
+                members: ['person.one', 'person.three']
+              }
+            }
+          ]
+        ]
+      )
+      
+      done()
+    })
+  })
+})

lib/modules/test/directPosts.js

@@ -0,0 +1,91 @@
+const expect = require('chai').expect
+const directPosts = require('../directPosts')
+const Fixtures = require('./fixtures')
+const FakeDB = require('./fakedb')
+const sink = require('./sink')
+const context = require('./context')()
+
+describe('modules.directPosts', function() {
+
+  before(function() {
+    //
+    // Set up context values
+    //
+    context.values = {
+      users: {
+        'person.one@example.com': {
+          username: 'person.one'
+        },
+        'person.two@example.com': {
+          username: 'person.two'
+        },
+        'person.three@example.com': {
+          username: 'person.three'
+        }
+      }
+    }
+  })
+
+  beforeEach(function() {
+    //
+    // Stub the output stream
+    //
+    context.output = sink()
+  })
+
+  it('should process direct channel objects', function(done) {
+    //
+    // Set up the DB
+    //
+    context.jabber = new FakeDB(Fixtures.directPosts)
+    //
+    // Process the data
+    //
+    directPosts(context).then(function(c) {
+      expect(c).to.equal(context)
+      expect(c.output.write.args).to.deep.equal([
+        [
+          {
+            type: 'direct_post',
+            direct_post: {
+              channel_members: [
+                'person.one', 'person.two'
+              ],
+              user: 'person.two',
+              message: 'message 01',
+              create_at: 1497066628513
+            }
+          }
+        ],
+        [
+          {
+            type: 'direct_post',
+            direct_post: {
+              channel_members: [
+                'person.one', 'person.three'
+              ],
+              user: 'person.three',
+              message: 'message 02',
+              create_at: 1497066628514
+            }
+          }
+        ],
+        [
+          {
+            type: 'direct_post',
+            direct_post: {
+              channel_members: [
+                'person.one', 'person.two'
+              ],
+              user: 'person.one',
+              message: 'message 03',
+              create_at: 1497066628515
+            }
+          }
+        ]
+      ])
+
+      done()
+    })
+  })
+})

lib/modules/test/end.js

@@ -0,0 +1,14 @@
+const expect = require('chai').expect
+const end = require('../end')
+const context = require('./context')()
+
+describe('modules.end', function() {
+  it('should close the output stream', function() {
+    end(context)
+    expect(context.output.end.called).to.be.true
+  })
+
+  afterEach(function() {
+    context.output.end.reset()
+  })
+})

lib/modules/test/fakedb.js

@@ -0,0 +1,50 @@
+//
+// Implements a fake db for testing
+//
+class FakeDB {
+  //
+  // Constructor
+  //
+  constructor(data) {
+    this.data = data
+    this.fetch = this.fetch.bind(this)
+    this.pipe = this.pipe.bind(this)
+  }
+
+  //
+  // Simulate a db connection by just returning
+  // a promise
+  //
+  connect() {
+    return Promise.resolve()
+  }
+
+  //
+  // Returns all results in one batch
+  //
+  fetch() {
+    return Promise.resolve({
+      recordset: this.data
+    })
+  }
+
+  //
+  // Delivers results to the specified write
+  // stream
+  //
+  pipe(query, writable) {
+
+    this.data.forEach(function(record) {
+      writable.write(record)
+    })
+
+    writable.end()
+
+    return writable
+  }
+}
+
+//
+// Export the class
+//
+module.exports = FakeDB

lib/modules/test/fixtures/channels.js

@@ -0,0 +1,15 @@
+module.exports = [
+  {
+    room_jid: 'admin@room.example.com',
+    subject: 'Admin Room',
+    config: '<x type=\'submit\' xmlns=\'jabber:x:data\'><field type=\'boolean\' var=\'persistent\'><value>1</value></field><field type=\'text-single\' var=\'max-persistent-history\'><value>15</value></field><field type=\'boolean\' var=\'open-membership\'><value>1</value></field><field type=\'boolean\' var=\'members-only\'><value>0</value></field><field type=\'boolean\' var=\'allow-gc\'><value>1</value></field><field type=\'boolean\' var=\'anonymous\'><value>1</value></field><field type=\'boolean\' var=\'show-unavailable\'><value>1</value></field><field type=\'list-single\' var=\'invite-role\'><value>participant</value></field><field type=\'boolean\' var=\'password\'><value>0</value></field><field type=\'text-private\' var=\'secret\'><value/></field><field type=\'list-single\' var=\'subject-acl\'><value>moderator</value></field><field type=\'boolean\' var=\'strip-xhtml\'><value>0</value></field><field type=\'boolean\' var=\'moderated-room\'><value>0</value></field></x>',
+  }, {
+    room_jid: 'test\\20room@room.example.com',
+    subject: '',
+    config: '<x type=\'submit\' xmlns=\'jabber:x:data\'><field type=\'boolean\' var=\'persistent\'><value>1</value></field><field type=\'text-single\' var=\'max-persistent-history\'><value>15</value></field><field type=\'boolean\' var=\'open-membership\'><value>1</value></field><field type=\'boolean\' var=\'members-only\'><value>0</value></field><field type=\'boolean\' var=\'allow-gc\'><value>1</value></field><field type=\'boolean\' var=\'anonymous\'><value>0</value></field><field type=\'boolean\' var=\'show-unavailable\'><value>1</value></field><field type=\'list-single\' var=\'invite-role\'><value>participant</value></field><field type=\'boolean\' var=\'password\'><value>0</value></field><field type=\'text-private\' var=\'secret\'><value/></field><field type=\'list-single\' var=\'subject-acl\'><value>moderator</value></field><field type=\'boolean\' var=\'strip-xhtml\'><value>0</value></field><field type=\'boolean\' var=\'moderated-room\'><value>0</value></field></x>',
+  }, {
+    room_jid: 'tonyd_20161206_1432@room.example.com',
+    subject: '',
+    config: '<x type=\'submit\' xmlns=\'jabber:x:data\'><field type=\'boolean\' var=\'persistent\'><value>1</value></field><field type=\'text-single\' var=\'max-persistent-history\'><value>15</value></field><field type=\'boolean\' var=\'open-membership\'><value>1</value></field><field type=\'boolean\' var=\'members-only\'><value>0</value></field><field type=\'boolean\' var=\'allow-gc\'><value>1</value></field><field type=\'boolean\' var=\'anonymous\'><value>0</value></field><field type=\'boolean\' var=\'show-unavailable\'><value>1</value></field><field type=\'list-single\' var=\'invite-role\'><value>participant</value></field><field type=\'boolean\' var=\'password\'><value>0</value></field><field type=\'text-private\' var=\'secret\'><value/></field><field type=\'list-single\' var=\'subject-acl\'><value>moderator</value></field><field type=\'boolean\' var=\'strip-xhtml\'><value>0</value></field><field type=\'boolean\' var=\'moderated-room\'><value>0</value></field></x>',
+  }
+]

lib/modules/test/fixtures/directChannels.js

@@ -0,0 +1,15 @@
+module.exports = [
+  {
+    to_jid: 'person.one@example.com/rnq1gmague',
+    from_jid: 'person.two@example.com/objc04v73q'
+  }, {
+    to_jid: 'person.one@example.com/rnq1gmague',
+    from_jid: 'person.three@example.com/w32ugtn6ch'
+  }, {
+    to_jid: 'person.two@example.com',
+    from_jid: 'person.one@example.com/blrjlefa3a'
+  }, {
+    to_jid: 'person.two@example.com/lhdms8czsw',
+    from_jid: 'person.one@example.com/blrjlefa3a'
+  }
+]

lib/modules/test/fixtures/directPosts.js

@@ -0,0 +1,27 @@
+module.exports = [
+  {
+    to_jid: 'person.one@example.com/Jabber Messenger Desktop',
+    from_jid: 'person.two@example.com/g1b9b8hs09',
+    sent_date: '2017-06-10T03:50:28.513Z',
+    body_string: 'message 01',
+    body_text: ''
+  }, {
+    to_jid: 'person.one@example.com/Jabber Messenger Desktop',
+    from_jid: 'person.three@example.com/g1b9b8hs09',
+    sent_date: '2017-06-10T03:50:28.514Z',
+    body_string: 'message 02',
+    body_text: ''
+  }, {
+    to_jid: 'person.two@example.com/g1b9b8hs09',
+    from_jid: 'person.one@example.com/Jabber Messenger Desktop',
+    sent_date: '2017-06-10T03:50:28.515Z',
+    body_string: '',
+    body_text: 'message 03'
+  }, {
+    to_jid: 'person.two@example.com/g1b9b8hs09',
+    from_jid: 'person.two@example.com/g1b9b8hs08',
+    sent_date: '2017-06-10T03:50:28.515Z',
+    body_string: 'This should not be exported',
+    body_text: ''
+  }
+]

lib/modules/test/fixtures/index.js

@@ -0,0 +1,13 @@
+const channels = require('./channels')
+const users = require('./users')
+const posts = require('./posts')
+const directChannels = require('./directChannels')
+const directPosts = require('./directPosts')
+
+module.exports = {
+  channels,
+  users,
+  posts,
+  directChannels,
+  directPosts
+}

lib/modules/test/fixtures/posts.js

@@ -0,0 +1,43 @@
+module.exports = {
+
+  ok: [{
+    msg_id: '3315',
+    to_jid: 'uat-appsupport@conference.example.com',
+    from_jid: 'micahel.cross@example.com',
+    sent_date: '2017-06-05T20:08:38.263Z',
+    body_string: 'I meant thick',
+    body_text: ''
+  }, {
+    msg_id: '3334',
+    to_jid: 'uat-appsupport@conference.example.com',
+    from_jid: 'micahel.cross@example.com',
+    sent_date: '2017-06-05T20:08:38.263Z',
+    body_string: 'that is when I came on again',
+    body_text: ''
+  }],
+
+  userNotFound: [{
+    msg_id: '3334',
+    to_jid: 'uat-appsupport@conference.example.com',
+    from_jid: 'terrence.flynn@example.com',
+    sent_date: '2017-06-05T20:08:38.263Z',
+    body_string: 'that is when I came on again',
+    body_text: ''
+  }, {
+    msg_id: '3315',
+    to_jid: 'uat-appsupport@conference.example.com',
+    from_jid: 'micahel.cross@example.com',
+    sent_date: '2017-06-05T20:08:38.263Z',
+    body_string: 'I meant thick',
+    body_text: ''
+  }],
+
+  bodyNotFound: [{
+    msg_id: '3315',
+    to_jid: 'uat-appsupport@conference.example.com',
+    from_jid: 'micahel.cross@example.com',
+    sent_date: '2017-06-05T20:08:38.263Z',
+    body_string: '',
+    body_text: ''
+  }]
+}

lib/modules/test/fixtures/users.js

@@ -0,0 +1,57 @@
+module.exports = [
+  {
+    room_jid: 'admin@conference.example.com',
+    real_jid: 'micahel.cross@example.com'
+  }, {
+    room_jid: 'admin@conference.example.com',
+    real_jid: 'sbarclay@example.com'
+  }, {
+    room_jid: 'uat-appsupport@conference.example.com',
+    real_jid: 'anthony.brown@example.com'
+  }, {
+    room_jid: 'uat-appsupport@conference.example.com',
+    real_jid: 'james.seegar1@example.com'
+  }, {
+    room_jid: 'uat-appsupport@conference.example.com',
+    real_jid: 'jarrett.jennings2@example.com'
+  }, {
+    room_jid: 'uat-appsupport@conference.example.com',
+    real_jid: 'micahel.cross@example.com'
+  }, {
+    room_jid: 'uat-appsupport@conference.example.com',
+    real_jid: 'michael.rhoades3@example.com'
+  }, {
+    room_jid: 'uat-appsupport@conference.example.com',
+    real_jid: 'michael.rock@example.com'
+  }, {
+    room_jid: 'uat-appsupport@conference.example.com',
+    real_jid: 'terrence.flynn@example.com'
+  }, {
+    room_jid: 'uat-appsupport@conference.example.com',
+    real_jid: 'tony.dilisio2@example.com'
+  }, {
+    room_jid: 'uat-appsupport@conference.example.com',
+    real_jid: 'william.fleming2@example.com'
+  }, {
+    room_jid: 'test\\20room@conference.example.com',
+    real_jid: 'lt.u755@example.com'
+  }, {
+    room_jid: 'test\\20room@conference.example.com',
+    real_jid: 'lt.u755@example.com/9fyg1zayoi'
+  }, {
+    room_jid: 'the\\20cool\\20room@conference.example.com',
+    real_jid: 'cody.webb@example.com'
+  }, {
+    room_jid: 'tonyd_20161206_1432@conference.example.com',
+    real_jid: 'tony.dilisio2@example.com'
+  }, {
+    room_jid: 'tonyd_20161206_1432@conference.example.com',
+    real_jid: 'julie..sokol@example.com'
+  }, {
+    real_jid: 'mbcross',
+    room_jid: null
+  },{
+    real_jid: 'cross,\\20michael@chat.dhs.gov',
+    room_jid: null
+  }
+]

lib/modules/test/posts.js

@@ -0,0 +1,131 @@
+const expect = require('chai').expect
+const posts = require('../posts')
+const Fixtures = require('./fixtures')
+const FakeDB = require('./fakedb')
+const sink = require('./sink')
+const context = require('./context')()
+
+describe('modules.posts', function() {
+
+  before(function() {
+    //
+    // Set up context values
+    //
+    context.values = {
+      team: context.config.define.team,
+      channels: {
+        'uat-appsupport@conference.example.com': {
+          team: 'test',
+          name: 'uat-appsupport',
+          display_name: 'Uat Appsupport',
+          header: '',
+          purpose: '',
+          type: 'P'
+        }
+      },
+      users: {
+        'micahel.cross@example.com': {
+          username: 'micahel.cross',
+          email: 'micahel.cross@example.com',
+          auth_service: 'ldap',
+          teams: [
+            {
+              name: 'test',
+              channels: [
+                {
+                  name: 'uat-appsupport'
+                }
+              ]
+            }
+          ]
+        }
+      }
+    }
+  })
+
+  beforeEach(function() {
+    //
+    // Stub the output stream
+    //
+    context.output = sink()
+  })
+
+  it('should process post objects', function(done) {
+    //
+    // Set up the DB
+    //
+    context.jabber = new FakeDB(Fixtures.posts.ok)
+    //
+    // Process the posts
+    //
+    posts(context).then(function(c) {
+      expect(c).to.equal(context)
+      expect(context.output.write.callCount).to.equal(2)
+
+      let post = c.output.write.args[0][0]
+      expect(post).to.deep.equal({
+        type: 'post',
+        post: {
+          team: 'test',
+          channel: 'uat-appsupport',
+          user: 'micahel.cross',
+          message: 'I meant thick',
+          create_at: 1496693318263
+        }
+      })
+      expect(new Date(post.post.create_at).toISOString()).to.equal('2017-06-05T20:08:38.263Z')
+      done()
+    }).catch(function(e){
+      expect(e).to.be.undefined
+    })
+  })
+
+
+  it('should fail on user not found', function(done) {
+    //
+    // Set up the DB
+    //
+    context.jabber = new FakeDB(Fixtures.posts.userNotFound)
+    //
+    // Process the posts
+    //
+    posts(context).then(function(c) {
+      expect(c).to.equal(context)
+      expect(c.output.write.callCount).to.equal(1)
+      let post = c.output.write.args[0][0]
+      expect(post).to.deep.equal({
+        type: 'post',
+        post: {
+          team: 'test',
+          channel: 'uat-appsupport',
+          user: 'micahel.cross',
+          message: 'I meant thick',
+          create_at: 1496693318263
+        }
+      })
+      done()
+    }).catch(function(e){
+      expect(e).to.be.null
+      done()
+    })
+  })
+
+
+  it('should fail on body not found', function(done) {
+    //
+    // Set up the DB
+    //
+    context.jabber = new FakeDB(Fixtures.posts.bodyNotFound)
+    //
+    // Process the posts
+    //
+    posts(context).then(function(c) {
+      expect(c).to.equal(context)
+      expect(c.output.write.callCount).to.equal(0)
+      done()
+    }).catch(function(e){
+      expect(e).to.be.null
+      done()
+    })
+  })
+})

lib/modules/test/sink.js

@@ -0,0 +1,18 @@
+const { spy } = require('sinon')
+const { Writable } = require('stream')
+
+//
+// Returns a spied writable stream
+//
+module.exports = function() {
+
+  var writable = new Writable({
+    objectMode: true,
+    write(chunk, encoding, callback) {
+      return callback()
+    }
+  })
+  spy(writable, 'write')
+
+  return writable
+}

lib/modules/test/start.js

@@ -0,0 +1,30 @@
+const expect = require('chai').expect
+const fs = require('fs')
+const start = require('../start')
+const context = require('./context')()
+
+describe('modules.start', function() {
+  var resultHandler = function(err) { 
+    if(err) {
+      console.log('unlink failed', err)
+    } else {
+      console.log('file deleted')
+    }
+  }
+  
+  it('should set up the source and output', function(done) {
+    start(context)
+      .then(function (c) {
+        expect(c).to.equal(context)
+        expect(fs.existsSync(context.config.target.filename)).to.be.true
+        done()
+      })
+      .catch(function(err){
+        console.log(err)
+      })
+  })
+
+  afterEach(function() {
+    fs.unlink(context.config.target.filename, resultHandler)
+  })
+})

lib/modules/test/team.js

@@ -0,0 +1,27 @@
+const expect = require('chai').expect
+const team = require('../team')
+const context = require('./context')()
+
+describe('modules.team', function() {
+  it('should write a team object', function(done) {
+    team(context)
+      .then(function (c) {
+        expect(c).to.equal(context)
+        expect(c.output.write.args[0][0]).to.deep.equal({
+          type: 'team',
+          team: {
+            name: 'test',
+            display_name: 'Test Team',
+            description: 'Our Test Team',
+            type: 'I',
+            allow_open_invite: false
+          }
+        })
+        done()
+      })
+  })
+
+  afterEach(function() {
+    context.output.write.reset()
+  })
+})

lib/modules/test/users.js

@@ -0,0 +1,88 @@
+const expect = require('chai').expect
+const users = require('../users')
+const Fixtures = require('./fixtures')
+const context = require('./context')()
+
+describe('modules.users', function() {
+
+  before(function(){
+    context.values = {
+      team: context.config.define.team,
+      channels: {
+        'admin@conference.example.com': {
+          team: 'test',
+          name: 'admin',
+          display_name: 'Admin',
+          header: 'Admin Test room',
+          purpose: 'Admin Test room',
+          type: 'O'
+        },
+        'uat-appsupport@conference.example.com': {
+          team: 'hsin',
+          name: 'uat-appsupport',
+          display_name: 'Uat Appsupport',
+          header: '',
+          purpose: '',
+          type: 'P'
+        }
+      }
+    }
+  })
+
+  it('should process user objects', function(done) {
+
+    context.jabber.fetch.returns(Promise.resolve({recordset: Fixtures.users}))
+
+    users(context).then(function(c) {
+      expect(c).to.equal(context)
+      expect(Object.keys(c.values.users).length).equals(13)
+      expect(c.output.write.args[0][0]).to.deep.equal({
+        type: 'user',
+        user: {
+          username: 'micahel.cross',
+          email: 'micahel.cross@example.com',
+          auth_service: 'ldap',
+          auth_data: 'MICAHEL.CROSS',
+          teams: [
+            {
+              name: 'test',
+              channels: [
+                {
+                  name: 'admin'
+                }, {
+                  name: 'uat-appsupport'
+                }
+              ]
+            }
+          ]
+        }
+      })
+      expect(c.output.write.args[1][0]).to.deep.equal({
+        type: 'user',
+        user: {
+          username: 'sbarclay',
+          email: 'sbarclay@example.com',
+          auth_service: 'ldap',
+          auth_data: 'SBARCLAY',
+          teams: [
+            {
+              name: 'test',
+              channels: [
+                {
+                  name: 'admin'
+                }
+              ]
+            }
+          ]
+        }
+      })
+
+      done()
+    })
+  })
+
+  afterEach(function() {
+    context.jabber.fetch.reset()
+    context.output.write.reset()
+  })
+})

lib/modules/test/version.js

@@ -0,0 +1,21 @@
+const expect = require('chai').expect
+const version = require('../version')
+const context = require('./context')()
+
+describe('modules.version', function() {
+  it('should write a version object', function(done) {
+    version(context)
+      .then(function (c) {
+        expect(c).to.equal(context)
+        expect(c.output.write.args[0][0]).to.deep.equal({
+          type: 'version',
+          version: 1
+        })
+        done()
+      })
+  })
+
+  afterEach(function() {
+    context.output.write.reset()
+  })
+})

lib/modules/transform.js

@@ -0,0 +1,9 @@
+const { Transform } = require('stream')
+
+module.exports = function(transform, callback) {
+  return new Transform({
+    readableObjectMode: true,
+    writableObjectMode: true,
+    transform
+  }).on('finish', callback)
+}

lib/modules/users.js

@@ -0,0 +1,115 @@
+const _ = require('lodash')
+const Factory = require('../factory')
+const Utils = require('./utils')
+
+//
+// Initialize the child logger for
+// the module
+//
+const log = require('../log').child({
+  module: 'users'
+})
+
+module.exports = function(context) {
+  //
+  // Select all of the users
+  //
+  const where = 'msg_type = \'c\''
+
+  return context.jabber.fetch(`
+      SELECT real_jid, room_jid FROM dbo.tc_users
+      UNION ALL (
+        SELECT from_jid AS real_jid, NULL AS room_jid FROM dbo.jm WHERE ${where}
+        UNION
+        SELECT to_jid AS real_jid, NULL AS room_jid FROM dbo.jm WHERE ${where}
+      )
+    `)
+    //
+    // Build up the user objects and then
+    // write them to the output
+    //
+    .then(function(results) {
+      log.info(`${results.recordset.length} records found`)
+      //
+      // Map of users
+      //
+      var users = {}
+      //
+      // Iterate over the record set and
+      // assemble the user objects
+      //
+      results.recordset.forEach(function(record) {
+        log.debug(record)
+        //
+        // Clean the real_jid to ensure we don't
+        // have duplicates with /<string> suffixes
+        //
+        var real_jid = Utils.realJID(record.real_jid)
+        //
+        // Generate the username fro the real_jid
+        //
+        var username = toUsername(real_jid)
+        //
+        // Return a reference to the user in the user map
+        // or add one if it doesn't yet exist
+        //
+        var user = users[real_jid] = _.get(users, real_jid, {
+          username,
+          email: real_jid,
+          auth_service: context.config.define.user.auth_service,
+          auth_data: username.toUpperCase(),
+          teams: [{
+            name: context.values.team.name,
+            channels: []
+          }]
+        })
+        //
+        // Look up the channel based on the
+        // room id. For direct messages, the room_jid
+        // will be null.
+        //
+        var channel = context.values.channels[record.room_jid]
+        //
+        // Add it to the user
+        //
+        if (channel) {
+          user.teams[0].channels = _.unionBy(user.teams[0].channels, [{
+            name: channel.name
+          }], 'name')
+        } else {
+          record.room_jid && log.warn(`... channel not found for ${record.room_jid}`)
+        }
+      })
+      //
+      // Now that the users are assembled, write
+      // them to the output
+      //
+      _.forEach(users, function(user, key) {
+        try {
+          context.output.write(
+            Factory.user(user)
+          )
+          log.info(`... writing ${user.username}`)
+        }
+        catch(err) {
+          log.error(`... ignoring ${user.username} on error: ${err.message}.`)
+          delete users[key]
+        }
+      })
+      //
+      // Add the users map to the context
+      //
+      context.values.users = users
+      //
+      // Return the context
+      //
+      return context
+    })
+}
+
+//
+// Parse the username
+//
+const toUsername = function (jid='') {
+  return jid.split('@')[0]
+}

lib/modules/utils.js

@@ -0,0 +1,103 @@
+const _ = require('lodash')
+
+//
+// Declare utils object
+//
+const utils = {}
+
+//
+//
+//
+utils.chunk = function(body) {
+  //
+  // Use regex to create an array
+  // of strings of max length
+  //
+  return body.match(/[\s\S]{1,4000}/g)
+}
+
+//
+// Removes trailing /<string> suffixes that
+// may exist in the user ids. This happens
+// if a user logs in to jabber more than
+// once at the same time
+//
+utils.realJID = function (jid='') {
+  return jid.split('/')[0]
+    .replace(/\\20+/g, '.')
+    .replace(/\.\.+/, '.')
+}
+
+//
+// Lookup values
+//
+utils.lookup = function (type, map, key) {
+  var found = map[key]
+
+  if(!found) {
+    throw new Error(`${type} ${key} not found`)
+  }
+
+  return found
+}
+
+//
+// Obtain the username from a jid
+//
+utils.username = function (users, jid) {
+  return utils.lookup('user', users, utils.realJID(jid)).username
+}
+
+//
+// Obtain the channel name from a jid
+//
+utils.channelName = function (channels, jid) {
+  return utils.lookup('channel', channels, jid).name
+}
+
+//
+// Find the message body
+//
+utils.body = function (message) {
+  var body = message.body_string || message.body_text
+
+  if(!body) {
+    throw new Error(`message ${message.msg_id} body is empty`)
+  }
+
+  return utils.chunk(body)
+}
+
+//
+// Coverts the to / from JIDs to a members
+// array
+//
+utils.members = function(users, to, from) {
+  //
+  // We use uniq to remove duplicate
+  // members in the same channel
+  //
+  return _.uniq(_.sortBy([
+    utils.username(users, to),
+    utils.username(users, from),
+  ]))
+}
+
+//
+// Checks if the members list is valid
+//
+utils.membersAreValid = function(members) {
+  return _.isArray(members) && members.length > 1
+}
+
+//
+// Convert ISO to millis
+//
+utils.millis = function(date) {
+  return new Date(date).getTime()
+}
+
+//
+// Export the functions
+//
+module.exports = utils

lib/modules/version.js

@@ -0,0 +1,12 @@
+const Factory = require('../factory')
+
+module.exports = function(context) {
+  //
+  // Write the version object
+  //
+  context.output.write(Factory.version())
+  //
+  // Return a resolved promise
+  //
+  return Promise.resolve(context)
+}