init

.codeclimate.yml

@@ -0,0 +1,17 @@
+engines:
+  eslint:
+    enabled: true
+  duplication:
+    enabled: true
+    config:
+      languages:
+      - javascript:
+  fixme:
+    enabled: true
+ratings:
+  paths:
+  - context/**
+  - lib/**
+  - "**.js"
+exclude_paths:
+- "**/test/**/*"

.eslintrc.yml

@@ -0,0 +1,20 @@
+env:
+  es6: true
+  node: true
+  mocha: true
+extends: 'eslint:recommended'
+rules:
+  indent:
+    - error
+    - 2
+  linebreak-style:
+    - error
+    - unix
+  quotes:
+    - error
+    - single
+  semi:
+    - error
+    - never
+  no-console:
+    - 0

.gitignore

@@ -0,0 +1,33 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# Coverage directory used by tools like istanbul
+coverage
+
+# nyc test coverage
+.nyc_output
+
+# Dependency directories
+node_modules/
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variables file
+.env
+
+# application config file
+context/config.js
+
+# generated data json file
+data.json

LICENSE

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2017 Brightscout, Inc.
+Copyright (c) 2026 ООО "Виликс"
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,67 @@
+# loop-etl-rocketchat
+
+An ETL framework to migrate data from RocketChat to Loop. This utility exports data from a source RocketChat database and generates a Loop import file.
+
+## Install
+
+1. Install [Node.js](https://nodejs.org/en/) Version [6.11.0 LTS](https://nodejs.org/en/download/) or greater
+
+2. Clone this repo  
+`$ git clone https://git.wilix.dev/loop/loop-etl-rocketchat`
+
+3. Install dependencies  
+`$ cd loop-etl-rocketchat`  
+`$ npm install`
+
+4. Run tests  
+`$ npm test`
+
+## Export RocketChat
+
+Export supports the following entities:
+
+1. Users and roles. Only global roles are supported
+2. Custom emoji. File storage: FileSystem and GridFS
+3. Channels
+4. Direct channels
+5. RocketChat Discussions (subchannels) are partially supported, see below
+7. User uploads. File storage: FileSystem and GridFS
+8. Posts with per user flag. Channel pins are not supported by import specification.
+9. Replies and Reactions
+
+## RocketChat exporting
+
+1. Copy the example config file to config.js  
+   ```
+   cp context/config.example.rocketchat.js context/config.js
+   ```
+
+2. Prepare your source and target configuration
+
+   1. Set `source.uploadsPath` for file uploads (user avatars and file uploads)
+   2. Set `source.customEmojiPath` for custom emojies
+   3. Set `target.filesPath` for Loop output directory
+
+3. If you have LDAP enabled, Community version of Loop doesn't support LDAP. 
+   If you have Community version, consider configuring ldap mapping to gitlab or disable it to use default login.  
+   
+   Specify it in `config.js`. Set `ldap_auth_service` to map ldap to a Loop login service. 
+   If it's GitLab, configure `gitlab` with `host` and `token` registered in Gitlab with User access. 
+   It is used for id mapping, without it Loop won't import the users and throw an error.  
+   Before migrating to Loop, ensure you have Gitlab integration enabled in Loop and all users are present in Gitlab.
+
+4. If you used RocketChat discussions, they will migrate in separate channels with random names. 
+   You can merge discussions in parent channel with `mergeDiscussionIntoParent`
+5. Global channel in Rocket Chat is **General** and in Loop - **Town Square**. 
+   To have only 1 global channel, the configuration provides default example in `channels.map`. 
+   You can specify migration for other channels as well.
+6. Run migrate script with `npm run start:rocketchat`
+7. Configure Loop (DB and Gitlab integration) before running the migration
+8. Run the migration in Loop
+
+## Import
+
+1. Run the Loop import command as explained in the [documentation](https://loop.yonote.ru/share/750208a7-e016-4319-8c7e-fb08a91f51f6)  
+---
+
+Based on [mattermost-etl](https://github.com/Brightscout/mattermost-etl) and [mattermost-etl-rocketchat](https://github.com/lexbritvin/mattermost-etl-rocketchat), both licensed under MIT.

circle.yml

@@ -0,0 +1,17 @@
+general:
+  artifacts:
+    - coverage
+
+machine:
+  node:
+    version: 6.11.0
+
+dependencies:
+  override:
+    - yarn
+
+test:
+  override:
+    - yarn run test-coverage
+  post:
+    - node_modules/codeclimate-test-reporter/bin/codeclimate.js < coverage/lcov.info

context/config.example.js

@@ -0,0 +1,36 @@
+module.exports = {
+  source: {
+    uri: 'mongodb://rocketchat:rocketchat@localhost:27017/rocketchat?replicaSet=rs0',
+    uploadsPath: './uploads',
+    customEmojiPath: './custom_emoji',
+  },
+  target: {
+    filename: './import.jsonl',
+    filesPath: './data/bulk-export-attachments'
+  },
+  define: {
+    team: {
+      // Уникальное системное имя команды (используется в URL, только латиница, цифры и дефисы)
+      name: 'loop',
+      // Отображаемое название команды, которое видят пользователи
+      display_name: 'loop',
+      // Краткое описание команды
+      description: 'An example of a team',
+      // Тип команды: 'I' — закрытая (invite-only), 'O' — открытая (open)
+      type: 'I',
+      // Разрешить пользователям приглашать других в команду без участия администратора
+      allow_open_invite: false
+    },
+    channels: {
+      mergeDiscussionIntoParent: true,
+    },
+    user: {
+      globalRoleMap: {
+        admin: 'system_admin',
+        user: 'system_user',
+      },
+      'auth_service': null,
+      'auth_data': null,
+    }
+  }
+}

context/index.js

@@ -0,0 +1,13 @@
+const config = require('./config')
+const jabber = require('../lib/mssql')
+
+module.exports = {
+  config,
+  jabber,
+  values: {
+    //
+    // Cached values can be stored
+    // here
+    //
+  }
+}

context/rocketchat.js

@@ -0,0 +1,38 @@
+const config = require('./config')
+const rocketchat = require('../lib/mongo')
+
+
+rocketchat.messagesCollection = function () {
+  return rocketchat.collection('rocketchat_message')
+}
+
+rocketchat.roomsCollection = function () {
+  return rocketchat.collection('rocketchat_room')
+}
+
+rocketchat.usersCollection = function () {
+  return rocketchat.collection('users')
+}
+
+rocketchat.uploadsCollection = function () {
+  return rocketchat.collection('rocketchat_uploads')
+}
+
+rocketchat.avatarsCollection = function () {
+  return rocketchat.collection('rocketchat_avatars')
+}
+
+rocketchat.emojiCollection = function () {
+  return rocketchat.collection('rocketchat_custom_emoji')
+}
+
+module.exports = {
+  config,
+  rocketchat,
+  values: {
+    //
+    // Cached values can be stored
+    // here
+    //
+  }
+}

index.js

@@ -0,0 +1,49 @@
+const context = require('./context')
+const log = require('./lib/log')
+const {
+  start,
+  version,
+  team,
+  channels,
+  users,
+  posts,
+  directChannels,
+  directPosts,
+  end
+} = require('./lib/modules')
+
+//
+// Common function log errors and
+// terminate the process
+//
+const abort = function(err) {
+  log.error(err)
+  //
+  // We set a timeout here to
+  // allow the log streams to finish
+  // writing.
+  //
+  setTimeout(function() {
+    process.exit(1)
+  }, 3000)
+}
+
+//
+// Ensure we trap uncaught exceptions
+// and properly abort
+//
+process.on('uncaughtException', abort)
+
+//
+// Let's do it
+//
+start(context)
+  .then(version)
+  .then(team)
+  .then(channels)
+  .then(users)
+  .then(posts)
+  .then(directChannels)
+  .then(directPosts)
+  .then(end)
+  .catch(abort)

lib/datafile.js

@@ -0,0 +1,31 @@
+const fs = require('fs')
+const os = require('os')
+const { Transform } = require('stream')
+
+module.exports = function(filename='data.json', onFinish=()=>{}) {
+  //
+  // Create an object to string transform
+  // stream
+  //
+  var stream = new Transform({
+    writableObjectMode: true,
+
+    transform(chunk, encoding, callback) {
+      this.push(JSON.stringify(chunk))
+      this.push(os.EOL)
+      return callback()
+    }
+  })
+  //
+  // Pipe that to the file write
+  // stream and set up an onFinish
+  // handler
+  //
+  stream.pipe(
+    fs.createWriteStream(filename).on('finish', onFinish)
+  )
+  //
+  // Return the new write stream
+  //
+  return stream
+}

lib/factory/channel.js

@@ -0,0 +1,24 @@
+const Joi = require('joi')
+const validate = require('./validate')
+
+//
+// Define the schema
+//
+const schema = {
+  team: Joi.string(),
+  name: Joi.string().regex(/^[a-z0-9_-]+$/),
+  display_name: Joi.string(),
+  header: Joi.string().allow('').optional(),
+  purpose: Joi.string().allow('').optional(),
+  type: Joi.string().valid('O', 'P')
+}
+
+//
+// Generate a valid object
+//
+module.exports = function (props) {
+  return {
+    type: 'channel',
+    channel: validate(schema, props)
+  }
+}

lib/factory/directChannel.js

@@ -0,0 +1,20 @@
+const Joi = require('joi')
+const validate = require('./validate')
+
+//
+// Define the schema
+//
+const schema = {
+  header: Joi.string().allow('').optional(),
+  members: Joi.array().items(Joi.string()).min(2).max(8)
+}
+
+//
+// Generate a valid object
+//
+module.exports = function (props) {
+  return {
+    type: 'direct_channel',
+    direct_channel: validate(schema, props)
+  }
+}

lib/factory/directPost.js

@@ -0,0 +1,21 @@
+const Joi = require('joi')
+const validate = require('./validate')
+const postPartial = require('./postPartial')
+
+//
+// Define the schema
+//
+const schema = {
+  channel_members: Joi.array().items(Joi.string()).min(2),
+  ...postPartial,
+}
+
+//
+// Generate a valid object
+//
+module.exports = function (props) {
+  return {
+    type: 'direct_post',
+    direct_post: validate(schema, props)
+  }
+}

lib/factory/emoji.js

@@ -0,0 +1,20 @@
+const Joi = require('joi')
+const validate = require('./validate')
+
+//
+// Define the schema
+//
+const schema = {
+  name: Joi.string(),
+  image: Joi.string(),
+}
+
+//
+// Generate a valid object
+//
+module.exports = function (props) {
+  return {
+    type: 'emoji',
+    emoji: validate(schema, props)
+  }
+}

lib/factory/index.js

@@ -0,0 +1,19 @@
+const version = require('./version')
+const emoji = require('./emoji')
+const team = require('./team')
+const channel = require('./channel')
+const user = require('./user')
+const post = require('./post')
+const directChannel = require('./directChannel')
+const directPost = require('./directPost')
+
+module.exports = {
+  version,
+  emoji,
+  team,
+  channel,
+  user,
+  post,
+  directChannel,
+  directPost
+}

lib/factory/post.js

@@ -0,0 +1,22 @@
+const Joi = require('joi')
+const validate = require('./validate')
+const postPartial = require('./postPartial')
+
+//
+// Define the schema
+//
+const schema = {
+  team: Joi.string(),
+  channel: Joi.string(),
+  ...postPartial,
+}
+
+//
+// Generate a valid object
+//
+module.exports = function (props) {
+  return {
+    type: 'post',
+    post: validate(schema, props)
+  }
+}

lib/factory/postPartial.js

@@ -0,0 +1,31 @@
+const Joi = require('joi')
+
+const attachment = Joi.object({
+  path: Joi.string()
+})
+
+const reaction = Joi.object({
+  user: Joi.string(),
+  emoji_name: Joi.string(),
+  create_at: Joi.number(),
+})
+
+const messagePartial = {
+  user: Joi.string(),
+  message: Joi.string().allow(''),
+  attachments: Joi.array().items(attachment).optional(),
+  flagged_by: Joi.array().items(Joi.string()).optional(),
+  reactions: Joi.array().items(reaction).optional(),
+  create_at: Joi.number(),
+}
+
+const reply = Joi.object().keys(messagePartial)
+
+//
+// Define the schema
+//
+module.exports = {
+  ...messagePartial,
+  replies: Joi.array().items(reply).optional(),
+}
+

lib/factory/team.js

@@ -0,0 +1,23 @@
+const Joi = require('joi')
+const validate = require('./validate')
+
+//
+// Define the schema
+//
+const schema = {
+  name: Joi.string(),
+  display_name: Joi.string(),
+  description: Joi.string(),
+  type: Joi.string().valid('O', 'I'),
+  allow_open_invite: Joi.boolean()
+}
+
+//
+// Generate a valid object
+//
+module.exports = function (props) {
+  return {
+    type: 'team',
+    team: validate(schema, props)
+  }
+}

lib/factory/test/channel.js

@@ -0,0 +1,35 @@
+const expect = require('chai').expect
+const channel = require('../channel')
+
+const basic = {
+  team: 'test-team',
+  name: 'test-channel',
+  display_name: 'Test Channel',
+  header: 'Test Channel Header',
+  purpose: 'Test the channel generator',
+  type: 'P'
+}
+
+describe('team factory', function() {
+
+  it('should produce a valid object', function() {
+    var c = channel(basic)
+    expect(c).to.be.an('object')
+    expect(c).to.deep.equal({
+      type: 'channel',
+      channel: basic
+    })
+  })
+
+  it('should prevent an invalid type', function() {
+    try {
+      channel(Object.assign({}, basic, {
+        type: 'X'
+      }))
+    }
+    catch (e) {
+      expect(e).to.be.an('error')
+      expect(e.details[0].message).to.equal('"type" must be one of [O, P]')
+    }
+  })
+})

lib/factory/test/directChannel.js

@@ -0,0 +1,35 @@
+const expect = require('chai').expect
+const directChannel = require('../directChannel')
+
+const basic = {
+  members: [
+    'username1',
+    'username2'
+  ]
+}
+
+describe('team factory', function() {
+
+  it('should produce a valid object', function() {
+    var c = directChannel(basic)
+    expect(c).to.be.an('object')
+    expect(c).to.deep.equal({
+      type: 'direct_channel',
+      direct_channel: basic
+    })
+  })
+
+  it('should prevent less than 2 members', function() {
+    try {
+      directChannel(Object.assign({}, basic, {
+        members:[
+          'username1'
+        ]
+      }))
+    }
+    catch (e) {
+      expect(e).to.be.an('error')
+      expect(e.details[0].message).to.equal('"members" must contain at least 2 items')
+    }
+  })
+})

lib/factory/test/directPost.js

@@ -0,0 +1,53 @@
+const expect = require('chai').expect
+const directPost = require('../directPost')
+
+const basic = {
+  channel_members: [
+    'username1',
+    'username2'
+  ],
+  user: 'username1',
+  message: 'carpe diem',
+  create_at: new Date().getTime()
+}
+
+describe('post factory', function() {
+
+  it('should produce a valid object', function() {
+    var p = directPost(basic)
+    expect(p).to.be.an('object')
+    expect(p).to.deep.equal({
+      type: 'direct_post',
+      direct_post: basic
+    })
+  })
+
+  it('should ensure message is not empty', function() {
+    try {
+      var p = directPost(Object.assign({}, basic, {
+        message: ''
+      }))
+      expect(p).to.be.undefined
+    }
+    catch (e) {
+      expect(e).to.be.an('error')
+      expect(e.details[0].message).to.equal('"message" is not allowed to be empty')
+    }
+  })
+
+  it('should prevent less than 2 members', function() {
+    try {
+      var p = directPost(Object.assign({}, basic, {
+        channel_members:[
+          'username1'
+        ]
+      }))
+      expect(p).to.be.undefined
+    }
+    catch (e) {
+      expect(e).to.be.an('error')
+      expect(e.details[0].message).to.equal('"channel_members" must contain at least 2 items')
+    }
+  })
+
+})

lib/factory/test/post.js

@@ -0,0 +1,35 @@
+const expect = require('chai').expect
+const post = require('../post')
+
+const basic = {
+  team: 'test',
+  channel: 'channel',
+  user: 'user',
+  message: 'carpe diem',
+  create_at: new Date().getTime()
+}
+
+describe('post factory', function() {
+
+  it('should produce a valid object', function() {
+    var p = post(basic)
+    expect(p).to.be.an('object')
+    expect(p).to.deep.equal({
+      type: 'post',
+      post: basic
+    })
+  })
+
+  it('should ensure message is not empty', function() {
+    try {
+      var p = post(Object.assign({}, basic, {
+        message: ''
+      }))
+      expect(p).to.be.undefined
+    }
+    catch (e) {
+      expect(e).to.be.an('error')
+      expect(e.details[0].message).to.equal('"message" is not allowed to be empty')
+    }
+  })
+})

lib/factory/test/team.js

@@ -0,0 +1,34 @@
+const expect = require('chai').expect
+const team = require('../team')
+
+const basic = {
+  name: 'test-team',
+  display_name: 'Test Team',
+  description: 'A test team for testing',
+  type: 'O',
+  allow_open_invite: true
+}
+
+describe('team factory', function() {
+
+  it('should produce a valid object', function() {
+    var t = team(basic)
+    expect(t).to.be.an('object')
+    expect(t).to.deep.equal({
+      type: 'team',
+      team: basic
+    })
+  })
+
+  it('should prevent an invalid type', function() {
+    try {
+      team(Object.assign({}, basic, {
+        type: 'X'
+      }))
+    }
+    catch (e) {
+      expect(e).to.be.an('error')
+      expect(e.details[0].message).to.equal('"type" must be one of [O, I]')
+    }
+  })
+})

lib/factory/test/user.js

@@ -0,0 +1,42 @@
+const expect = require('chai').expect
+const user = require('../user')
+
+const basic = {
+  username: 'user.name',
+  email: 'user@example.gov',
+  auth_service: 'ldap',
+  auth_data: 'username-field-reference',
+  teams: [{
+    name: 'test',
+    channels: [{
+      name: 'channel-1'
+    }, {
+      name: 'channel-2'
+    }]
+  }]
+}
+
+describe('user factory', function() {
+
+  it('should produce a valid object', function() {
+    var u = user(basic)
+    expect(u).to.be.an('object')
+    expect(u).to.deep.equal({
+      type: 'user',
+      user: basic
+    })
+  })
+
+  it('should ensure email is valid', function() {
+    try {
+      var u = user(Object.assign({}, basic, {
+        email: 'foo@bar'
+      }))
+      expect(u).to.be.undefined
+    }
+    catch (e) {
+      expect(e).to.be.an('error')
+      expect(e.details[0].message).to.equal('"email" must be a valid email')
+    }
+  })
+})

lib/factory/test/version.js

@@ -0,0 +1,13 @@
+const expect = require('chai').expect
+const version = require('../version')
+
+describe('version factory', function() {
+  it('should produce a valid version', function() {
+    var v = version()
+    expect(v).to.be.an('object')
+    expect(v).to.deep.equal({
+      type: 'version',
+      version: 1
+    })
+  })
+})

lib/factory/user.js

@@ -0,0 +1,72 @@
+const Joi = require('joi')
+const validate = require('./validate')
+
+const validateRoles = function (value, helper, allowed) {
+  const roles = value.split(' ')
+  if (value.length < 8) {
+    return helper.message('Password must be at least 8 characters long')
+  } else {
+    return true
+  }
+}
+
+//
+// Define the schema
+//
+const schema = {
+  profile_image: Joi.string().optional(),
+  username: Joi.string(),
+  email: Joi.string().email({
+    errorLevel: true,
+    minDomainAtoms: 2
+  }),
+  auth_service: Joi.string().valid(
+    '',
+    'gitlab',
+    'ldap',
+    'saml',
+    'google',
+    'office365'
+  ).optional(),
+  auth_data: Joi.string().optional().allow(''),
+  password: Joi.string().optional(),
+  nickname: Joi.string().optional(),
+  first_name: Joi.string().optional(),
+  last_name: Joi.string().optional(),
+  position: Joi.string().optional(),
+  roles: Joi.string().optional().valid(
+    'system_user',
+    'system_admin system_user'
+  ),
+  teams: Joi.array().items(
+    Joi.object({
+      name: Joi.string(),
+      roles: Joi.string().optional().valid(
+        'team_user',
+        'team_admin team_user'
+      ),
+      channels: Joi.array().items(
+        Joi.object({
+          name: Joi.string(),
+          roles: Joi.string().optional().valid(
+            'channel_user',
+            'channel_user channel_admin'
+          )
+        })
+      )
+    })
+  ),
+  notify_props: Joi.object({
+    mention_keys: Joi.string().optional().allow(''),
+  }).optional(),
+}
+
+//
+// Generate a valid object
+//
+module.exports = function (props) {
+  return {
+    type: 'user',
+    user: validate(schema, props)
+  }
+}

lib/factory/validate.js

@@ -0,0 +1,21 @@
+const Joi = require('joi')
+
+module.exports = function (schema, props) {
+  //
+  //  Validate and remove unknown keys
+  //
+  var {error, value} = Joi.validate(props, schema, {
+    presence: 'required',
+    stripUnknown: true
+  })
+  //
+  // Throw validation errors
+  //
+  if (error) {
+    throw error
+  }
+  //
+  // Return the value
+  //
+  return value
+}

lib/factory/version.js

@@ -0,0 +1,11 @@
+//
+// Generate a version object. This
+// is the only valid value as of
+// now
+//
+module.exports = function() {
+  return {
+    type: 'version',
+    version: 1
+  }
+}

lib/log.js

@@ -0,0 +1,15 @@
+const bunyan = require('bunyan')
+
+module.exports = bunyan.createLogger({
+  name: 'mm-etl',
+  streams: [
+    {
+      level: 'info',
+      stream: process.stdout
+    },
+    {
+      level: 'info',
+      path: 'mm-etl.log'
+    }
+  ]
+})

lib/modules/channels.js

@@ -0,0 +1,114 @@
+const _ = require('lodash')
+const slug = require('slug')
+const XML = require('xmldoc')
+const Factory = require('../factory')
+
+//
+// Initialize the child logger for
+// the module
+//
+const log = require('../log').child({
+  module: 'channels'
+})
+
+module.exports = function(context) {
+  //
+  // Select all of the rooms
+  //
+  return context.jabber.fetch(
+    'SELECT room_jid, subject, config FROM dbo.tc_rooms'
+  )
+    //
+    // The convert them to channels and
+    // write them to the datafile
+    //
+    .then(function(results) {
+      log.info(`${results.recordset.length} records found`)
+      //
+      // Define a map to hold the channels
+      // info for downstream modules
+      //
+      var channels = {}
+      //
+      // Iterate over the record set and
+      // create a channel for each room
+      //
+      results.recordset.forEach(function(room) {
+        log.debug(room)
+        //
+        // Define the channel and add
+        // it to the context
+        //
+        var channel = channels[room.room_jid] = {
+          team: context.values.team.name,
+          name: slug(toName(room.room_jid)),
+          display_name: toDisplayName(room.room_jid),
+          header: toDescription(room.subject),
+          purpose: toDescription(room.subject),
+          type: toType(room.config)
+        }
+        //
+        // Write the channel data to the
+        // output
+        //
+        log.info(`... writing ${channel.name}`)
+        context.output.write(
+          Factory.channel(channel)
+        )
+      })
+      //
+      // Add the channel map to the context
+      //
+      context.values.channels = channels
+      //
+      // Resolve the promise
+      //
+      return context
+    })
+}
+
+//
+// Parses the name from a room jid
+//
+const toName = function (jid='') {
+  return jid.substr(0, jid.indexOf('@')).replace(/\\20/g, ' ')
+}
+
+//
+// Formats the display name
+//
+const toDisplayName = function (jid='') {
+  return _.startCase(_.toLower(toName(jid)))
+}
+
+//
+// Returns a description from the subject
+//
+const toDescription = function (subject='') {
+  return subject
+}
+
+//
+// Parses the channel type from the
+// config field
+//
+const toType = function (xml='<x/>') {
+  //
+  // Parse the config
+  //
+  var config = new XML.XmlDocument(xml)
+  //
+  // Extract the value. If it does not exist,
+  // we default to '1' - private
+  //
+  var value = _.get(
+    config.childWithAttribute('var', 'members-only'),
+    'firstChild.val',
+    '1'
+  )
+  //
+  // If members-only is '1', return private
+  // otherwise, return public
+  //
+  return value === '1' ? 'P' : 'O'
+}

lib/modules/directChannels.js

@@ -0,0 +1,89 @@
+const Factory = require('../factory')
+const transform = require('./transform')
+const Utils = require('./utils')
+
+//
+// Initialize the child logger for
+// the module
+//
+const log = require('../log').child({
+  module: 'directChannels'
+})
+
+module.exports = function(context) {
+  return new Promise(function(resolve /*, reject */) {
+    log.info('streaming records')
+    //
+    // Array to accumulate the channel
+    // member pairs
+    //
+    var channels = {}
+    //
+    // Query messages from Jabber and pipe
+    // through the post transform and
+    // then to the output. We use pipe to
+    // handle very large data sets using
+    // streams
+    //
+    context.jabber.pipe(
+      //
+      // Define the query
+      //
+      `SELECT DISTINCT to_jid, from_jid FROM dbo.jm
+        WHERE msg_type = 'c'
+          AND direction = 'I'
+          AND (body_string != '' or datalength(body_text) > 0)`,
+      //
+      // Define the tranform
+      //
+      transform(function(result, encoding, callback) {
+        try {
+          log.debug(result)
+          //
+          // Generate the members array for the
+          // direct channel
+          //
+          let members = Utils.members(
+            context.values.users,
+            result.to_jid,
+            result.from_jid
+          )
+          //
+          // If there at least two members
+          //
+          if(Utils.membersAreValid(members)) {
+            var key = `${members[0]}|${members[1]}`
+            //
+            // And we haven't processed this pair
+            // already
+            //
+            if(!channels[key]) {
+              channels[key] = members
+              log.info(`... writing ${members}`)
+              context.output.write(
+                Factory.directChannel({
+                  members
+                })
+              )
+            }
+          }
+        } catch (err) {
+          log.error(`... ignoring directChannel from: ${result.from_jid} to: ${result.to_jid} on error: ${err.message}.`)
+        }
+        //
+        // Invoke the call to mark that we are
+        // done with the chunk
+        //
+        return callback()
+      },
+      //
+      // Define the callback to be invoked
+      // on finish
+      //
+      function() {
+        log.info('... finished')
+        resolve(context)
+      })
+    )
+  })
+}

lib/modules/directPosts.js

@@ -0,0 +1,106 @@
+const Factory = require('../factory')
+const transform = require('./transform')
+const Utils = require('./utils')
+
+//
+// Initialize the child logger for
+// the module
+//
+const log = require('../log').child({
+  module: 'directPosts'
+})
+
+module.exports = function(context) {
+  return new Promise(function(resolve /*, reject */) {
+    log.info('streaming records')
+    //
+    // Keep track of the number of posts
+    // written for logging
+    //
+    var written = 0
+    //
+    // Query messages from Jabber and pipe
+    // through the post transform and
+    // then to the output. We use pipe to
+    // handle very large data sets using
+    // streams
+    //
+    context.jabber.pipe(
+      //
+      // Define the query
+      //
+      `SELECT to_jid, from_jid, sent_date, body_string, body_text FROM dbo.jm
+          WHERE msg_type = 'c'
+            AND direction = 'I'
+            AND (body_string != '' or datalength(body_text) > 0)`,
+      //
+      // Define the tranform
+      //
+      transform(function(message, encoding, callback) {
+        try {
+          log.debug(message)
+          //
+          // Generate the members array for the
+          // direct channel
+          //
+          let members = Utils.members(
+            context.values.users,
+            message.to_jid,
+            message.from_jid
+          )
+          //
+          // Ensure we have at least two channel
+          // members before we can write the message
+          //
+          if (Utils.membersAreValid(members)) {
+            //
+            // Base post props
+            //
+            var post = {
+              channel_members: members,
+              user: Utils.username(context.values.users, message.from_jid),
+              create_at: Utils.millis(message.sent_date)
+            }
+            //
+            // Process each chunk
+            //
+            Utils.body(message).forEach(function(chunk) {
+              context.output.write(
+                Factory.directPost(Object.assign({}, post, {
+                  message: chunk
+                }))
+              )
+              //
+              // Log progress periodically
+              //
+              written += 1
+              if(written % 1000 == 0) {
+                log.info(`... wrote ${written} posts`)
+              }
+            })
+          } else {
+            log.warn({
+              to_jid: message.to_jid,
+              from_jid: message.from_jid
+            }, '... skipping message with invalid channel members')
+          }
+        } catch (err) {
+          log.error(`... ignoring directPost from: ${message.from_jid} to: ${message.to_jid} on error: ${err.message}.`)
+        }
+        //
+        // Invoke the call to mark that we are
+        // done with the chunk
+        //
+        return callback()
+      },
+      //
+      // Define the callback to be invoked
+      // on finish
+      //
+      function() {
+        log.info(`... finished writing ${written} posts`)
+        resolve(context)
+      })
+    )
+  })
+}

lib/modules/end.js

@@ -0,0 +1,12 @@
+//
+// Initialize the child logger for
+// the module
+//
+const log = require('../log').child({
+  module: 'end'
+})
+
+module.exports = function(context) {
+  log.info('end')
+  context.output.end()
+}

lib/modules/index.js

@@ -0,0 +1,21 @@
+const start = require('./start')
+const version = require('./version')
+const team = require('./team')
+const channels = require('./channels')
+const users = require('./users')
+const posts = require('./posts')
+const directChannels = require('./directChannels')
+const directPosts = require('./directPosts')
+const end = require('./end')
+
+module.exports = {
+  start,
+  version,
+  team,
+  channels,
+  users,
+  posts,
+  directChannels,
+  directPosts,
+  end
+}

lib/modules/posts.js

@@ -0,0 +1,100 @@
+const Factory = require('../factory')
+const transform = require('./transform')
+const Utils = require('./utils')
+
+//
+// Initialize the child logger for
+// the module
+//
+const log = require('../log').child({
+  module: 'posts'
+})
+
+module.exports = function(context) {
+  return new Promise(function(resolve /*, reject */) {
+    log.info('streaming records')
+    //
+    // Keep track of the number of posts
+    // written for logging
+    //
+    var written = 0
+    //
+    // Query messages from Jabber and pipe
+    // through the post transform and
+    // then to the output. We use pipe to
+    // handle very large data sets using
+    // streams
+    //
+    context.jabber.pipe(
+      //
+      // Define the query
+      //
+      `SELECT
+            msg_id,
+            to_jid,
+            from_jid,
+            sent_date,
+            body_string,
+            body_text
+        FROM dbo.tc_msgarchive WHERE msg_type = 'g' AND (body_string != '' OR datalength(body_text) > 0) `,
+      //
+      // Define the tranform
+      //
+      transform(function(message, encoding, callback) {
+        try {
+          log.debug(message)
+          //
+          // Base post props
+          //
+          var post = {
+            team: context.values.team.name,
+            channel: Utils.channelName(
+              context.values.channels, message.to_jid
+            ),
+            user: Utils.username(
+              context.values.users, message.from_jid
+            ),
+            create_at: Utils.millis(message.sent_date)
+          }
+          //
+          // Process each chunk
+          //
+          Utils.body(message).forEach(function(chunk) {
+            //
+            // Write the post object to the
+            // output
+            //
+            context.output.write(
+              Factory.post(Object.assign({}, post, {
+                message: chunk
+              }))
+            )
+            //
+            // Log progress periodically
+            //
+            written += 1
+            if(written % 1000 == 0) {
+              log.info(`... wrote ${written} posts`)
+            }
+          })
+        }
+        catch(err) {
+          log.error(`... ignoring message id:${message.msg_id} on error: ${err.message}.`)
+        }
+        //
+        // Invoke the call to mark that we are
+        // done with the chunk
+        //
+        return callback()
+      },
+      //
+      // Define the callback to be invoked
+      // on finish
+      //
+      function() {
+        log.info(`... finished writing ${written} posts`)
+        resolve(context)
+      })
+    )
+  })
+}

lib/modules/start.js

@@ -0,0 +1,32 @@
+const datafile = require('../datafile')
+
+//
+// Initialize the child logger for
+// the module
+//
+const log = require('../log').child({
+  module: 'start'
+})
+
+module.exports = function(context) {
+  log.info('connecting to jabber')
+  //
+  // Connect to the jabber database
+  //
+  return context.jabber.connect(context.config)
+    .then(function() {
+      log.info(`creating file '${context.config.target.filename}'`)
+      //
+      // Create the datafile and add
+      // it to the context
+      //
+      context.output = datafile(
+        context.config.target.filename,
+        process.exit
+      )
+      //
+      // Resolve the promise
+      //
+      return context
+    })
+}

lib/modules/team.js

@@ -0,0 +1,29 @@
+const Factory = require('../factory')
+
+//
+// Initialize the child logger for
+// the module
+//
+const log = require('../log').child({
+  module: 'team'
+})
+
+module.exports = function(context) {
+  log.info(`writing team '${context.config.define.team.name}'`)
+  //
+  // Store the team object for use
+  // by other modules
+  //
+  context.values.team = context.config.define.team
+  //
+  // Write the team object to the
+  // output
+  //
+  context.output.write(Factory.team(
+    context.config.define.team
+  ))
+  //
+  // Return a resolved promise
+  //
+  return Promise.resolve(context)
+}

lib/modules/test/channels.js

@@ -0,0 +1,61 @@
+const expect = require('chai').expect
+const channels = require('../channels')
+const Fixtures = require('./fixtures')
+const context = require('./context')()
+
+describe('modules.channels', function() {
+
+  it('should process channel objects', function(done) {
+
+    context.values.team = context.config.define.team
+
+    context.jabber.fetch.returns(Promise.resolve({
+      recordset: Fixtures.channels
+    }))
+
+    channels(context)
+      .then(function (c) {
+        expect(c).to.equal(context)
+        expect(Object.keys(c.values.channels).length).equals(3)
+        expect(c.output.write.args[0][0]).to.deep.equal({
+          type: 'channel',
+          channel: {
+            team: 'test',
+            name: 'admin',
+            display_name: 'Admin',
+            header: 'Admin Room',
+            purpose: 'Admin Room',
+            type: 'O'
+          }
+        })
+        expect(c.output.write.args[1][0]).to.deep.equal({
+          type: 'channel',
+          channel: {
+            team: 'test',
+            name: 'test-room',
+            display_name: 'Test Room',
+            header: '',
+            purpose: '',
+            type: 'O'
+          }
+        })
+        expect(c.output.write.args[2][0]).to.deep.equal({
+          type: 'channel',
+          channel: {
+            team: 'test',
+            name: 'tonyd_20161206_1432',
+            display_name: 'Tonyd 20161206 1432',
+            header: '',
+            purpose: '',
+            type: 'O'
+          }
+        })
+        done()
+      })
+  })
+
+  afterEach(function() {
+    context.jabber.fetch.reset()
+    context.output.write.reset()
+  })
+})

lib/modules/test/context.js

@@ -0,0 +1,38 @@
+const { spy, stub } = require('sinon')
+
+module.exports = function() {
+  return {
+    config: {
+      source: {
+        uri: 'mssql://username:password@server:1433/jabber?encrypt=true'
+      },
+      target: {
+        filename: 'data.json'
+      },
+      define: {
+        team: {
+          name: 'test',
+          display_name: 'Test Team',
+          description: 'Our Test Team',
+          type: 'I',
+          allow_open_invite: false
+        },
+        user: {
+          auth_service: 'ldap',
+        }
+      }
+    },
+    jabber: {
+      connect: stub().returns(Promise.resolve()),
+      fetch: stub(),
+      pipe: stub()
+    },
+    values: {
+
+    },
+    output: {
+      write: spy(),
+      end: spy()
+    }
+  }
+}

lib/modules/test/directChannels.js

@@ -0,0 +1,70 @@
+const expect = require('chai').expect
+const directChannels = require('../directChannels')
+const Fixtures = require('./fixtures')
+const FakeDB = require('./fakedb')
+const sink = require('./sink')
+const context = require('./context')()
+
+describe('modules.directChannels', function() {
+
+  before(function() {
+    //
+    // Set up context values
+    //
+    context.values = {
+      users: {
+        'person.one@example.com': {
+          username: 'person.one'
+        },
+        'person.two@example.com': {
+          username: 'person.two'
+        },
+        'person.three@example.com': {
+          username: 'person.three'
+        }
+      }
+    }
+  })
+
+  beforeEach(function() {
+    //
+    // Stub the output stream
+    //
+    context.output = sink()
+  })
+
+  it('should process direct channel objects', function(done) {
+    //
+    // Set up the DB
+    //
+    context.jabber = new FakeDB(Fixtures.directChannels)
+    //
+    // Process the data
+    //
+    directChannels(context).then(function(c) {
+      expect(c).to.equal(context)
+      expect(c.output.write.args).to.deep.equal(
+        [
+          [
+            {
+              type: 'direct_channel',
+              direct_channel: {
+                members: ['person.one', 'person.two']
+              }
+            }
+          ],
+          [
+            {
+              type: 'direct_channel',
+              direct_channel: {
+                members: ['person.one', 'person.three']
+              }
+            }
+          ]
+        ]
+      )
+      
+      done()
+    })
+  })
+})

lib/modules/test/directPosts.js

@@ -0,0 +1,91 @@
+const expect = require('chai').expect
+const directPosts = require('../directPosts')
+const Fixtures = require('./fixtures')
+const FakeDB = require('./fakedb')
+const sink = require('./sink')
+const context = require('./context')()
+
+describe('modules.directPosts', function() {
+
+  before(function() {
+    //
+    // Set up context values
+    //
+    context.values = {
+      users: {
+        'person.one@example.com': {
+          username: 'person.one'
+        },
+        'person.two@example.com': {
+          username: 'person.two'
+        },
+        'person.three@example.com': {
+          username: 'person.three'
+        }
+      }
+    }
+  })
+
+  beforeEach(function() {
+    //
+    // Stub the output stream
+    //
+    context.output = sink()
+  })
+
+  it('should process direct channel objects', function(done) {
+    //
+    // Set up the DB
+    //
+    context.jabber = new FakeDB(Fixtures.directPosts)
+    //
+    // Process the data
+    //
+    directPosts(context).then(function(c) {
+      expect(c).to.equal(context)
+      expect(c.output.write.args).to.deep.equal([
+        [
+          {
+            type: 'direct_post',
+            direct_post: {
+              channel_members: [
+                'person.one', 'person.two'
+              ],
+              user: 'person.two',
+              message: 'message 01',
+              create_at: 1497066628513
+            }
+          }
+        ],
+        [
+          {
+            type: 'direct_post',
+            direct_post: {
+              channel_members: [
+                'person.one', 'person.three'
+              ],
+              user: 'person.three',
+              message: 'message 02',
+              create_at: 1497066628514
+            }
+          }
+        ],
+        [
+          {
+            type: 'direct_post',
+            direct_post: {
+              channel_members: [
+                'person.one', 'person.two'
+              ],
+              user: 'person.one',
+              message: 'message 03',
+              create_at: 1497066628515
+            }
+          }
+        ]
+      ])
+
+      done()
+    })
+  })
+})

lib/modules/test/end.js

@@ -0,0 +1,14 @@
+const expect = require('chai').expect
+const end = require('../end')
+const context = require('./context')()
+
+describe('modules.end', function() {
+  it('should close the output stream', function() {
+    end(context)
+    expect(context.output.end.called).to.be.true
+  })
+
+  afterEach(function() {
+    context.output.end.reset()
+  })
+})

lib/modules/test/fakedb.js

@@ -0,0 +1,50 @@
+//
+// Implements a fake db for testing
+//
+class FakeDB {
+  //
+  // Constructor
+  //
+  constructor(data) {
+    this.data = data
+    this.fetch = this.fetch.bind(this)
+    this.pipe = this.pipe.bind(this)
+  }
+
+  //
+  // Simulate a db connection by just returning
+  // a promise
+  //
+  connect() {
+    return Promise.resolve()
+  }
+
+  //
+  // Returns all results in one batch
+  //
+  fetch() {
+    return Promise.resolve({
+      recordset: this.data
+    })
+  }
+
+  //
+  // Delivers results to the specified write
+  // stream
+  //
+  pipe(query, writable) {
+
+    this.data.forEach(function(record) {
+      writable.write(record)
+    })
+
+    writable.end()
+
+    return writable
+  }
+}
+
+//
+// Export the class
+//
+module.exports = FakeDB

lib/modules/test/fixtures/channels.js

@@ -0,0 +1,15 @@
+module.exports = [
+  {
+    room_jid: 'admin@room.example.com',
+    subject: 'Admin Room',
+    config: '<x type=\'submit\' xmlns=\'jabber:x:data\'><field type=\'boolean\' var=\'persistent\'><value>1</value></field><field type=\'text-single\' var=\'max-persistent-history\'><value>15</value></field><field type=\'boolean\' var=\'open-membership\'><value>1</value></field><field type=\'boolean\' var=\'members-only\'><value>0</value></field><field type=\'boolean\' var=\'allow-gc\'><value>1</value></field><field type=\'boolean\' var=\'anonymous\'><value>1</value></field><field type=\'boolean\' var=\'show-unavailable\'><value>1</value></field><field type=\'list-single\' var=\'invite-role\'><value>participant</value></field><field type=\'boolean\' var=\'password\'><value>0</value></field><field type=\'text-private\' var=\'secret\'><value/></field><field type=\'list-single\' var=\'subject-acl\'><value>moderator</value></field><field type=\'boolean\' var=\'strip-xhtml\'><value>0</value></field><field type=\'boolean\' var=\'moderated-room\'><value>0</value></field></x>',
+  }, {
+    room_jid: 'test\\20room@room.example.com',
+    subject: '',
+    config: '<x type=\'submit\' xmlns=\'jabber:x:data\'><field type=\'boolean\' var=\'persistent\'><value>1</value></field><field type=\'text-single\' var=\'max-persistent-history\'><value>15</value></field><field type=\'boolean\' var=\'open-membership\'><value>1</value></field><field type=\'boolean\' var=\'members-only\'><value>0</value></field><field type=\'boolean\' var=\'allow-gc\'><value>1</value></field><field type=\'boolean\' var=\'anonymous\'><value>0</value></field><field type=\'boolean\' var=\'show-unavailable\'><value>1</value></field><field type=\'list-single\' var=\'invite-role\'><value>participant</value></field><field type=\'boolean\' var=\'password\'><value>0</value></field><field type=\'text-private\' var=\'secret\'><value/></field><field type=\'list-single\' var=\'subject-acl\'><value>moderator</value></field><field type=\'boolean\' var=\'strip-xhtml\'><value>0</value></field><field type=\'boolean\' var=\'moderated-room\'><value>0</value></field></x>',
+  }, {
+    room_jid: 'tonyd_20161206_1432@room.example.com',
+    subject: '',
+    config: '<x type=\'submit\' xmlns=\'jabber:x:data\'><field type=\'boolean\' var=\'persistent\'><value>1</value></field><field type=\'text-single\' var=\'max-persistent-history\'><value>15</value></field><field type=\'boolean\' var=\'open-membership\'><value>1</value></field><field type=\'boolean\' var=\'members-only\'><value>0</value></field><field type=\'boolean\' var=\'allow-gc\'><value>1</value></field><field type=\'boolean\' var=\'anonymous\'><value>0</value></field><field type=\'boolean\' var=\'show-unavailable\'><value>1</value></field><field type=\'list-single\' var=\'invite-role\'><value>participant</value></field><field type=\'boolean\' var=\'password\'><value>0</value></field><field type=\'text-private\' var=\'secret\'><value/></field><field type=\'list-single\' var=\'subject-acl\'><value>moderator</value></field><field type=\'boolean\' var=\'strip-xhtml\'><value>0</value></field><field type=\'boolean\' var=\'moderated-room\'><value>0</value></field></x>',
+  }
+]

lib/modules/test/fixtures/directChannels.js

@@ -0,0 +1,15 @@
+module.exports = [
+  {
+    to_jid: 'person.one@example.com/rnq1gmague',
+    from_jid: 'person.two@example.com/objc04v73q'
+  }, {
+    to_jid: 'person.one@example.com/rnq1gmague',
+    from_jid: 'person.three@example.com/w32ugtn6ch'
+  }, {
+    to_jid: 'person.two@example.com',
+    from_jid: 'person.one@example.com/blrjlefa3a'
+  }, {
+    to_jid: 'person.two@example.com/lhdms8czsw',
+    from_jid: 'person.one@example.com/blrjlefa3a'
+  }
+]

lib/modules/test/fixtures/directPosts.js

@@ -0,0 +1,27 @@
+module.exports = [
+  {
+    to_jid: 'person.one@example.com/Jabber Messenger Desktop',
+    from_jid: 'person.two@example.com/g1b9b8hs09',
+    sent_date: '2017-06-10T03:50:28.513Z',
+    body_string: 'message 01',
+    body_text: ''
+  }, {
+    to_jid: 'person.one@example.com/Jabber Messenger Desktop',
+    from_jid: 'person.three@example.com/g1b9b8hs09',
+    sent_date: '2017-06-10T03:50:28.514Z',
+    body_string: 'message 02',
+    body_text: ''
+  }, {
+    to_jid: 'person.two@example.com/g1b9b8hs09',
+    from_jid: 'person.one@example.com/Jabber Messenger Desktop',
+    sent_date: '2017-06-10T03:50:28.515Z',
+    body_string: '',
+    body_text: 'message 03'
+  }, {
+    to_jid: 'person.two@example.com/g1b9b8hs09',
+    from_jid: 'person.two@example.com/g1b9b8hs08',
+    sent_date: '2017-06-10T03:50:28.515Z',
+    body_string: 'This should not be exported',
+    body_text: ''
+  }
+]

lib/modules/test/fixtures/index.js

@@ -0,0 +1,13 @@
+const channels = require('./channels')
+const users = require('./users')
+const posts = require('./posts')
+const directChannels = require('./directChannels')
+const directPosts = require('./directPosts')
+
+module.exports = {
+  channels,
+  users,
+  posts,
+  directChannels,
+  directPosts
+}

lib/modules/test/fixtures/posts.js

@@ -0,0 +1,43 @@
+module.exports = {
+
+  ok: [{
+    msg_id: '3315',
+    to_jid: 'uat-appsupport@conference.example.com',
+    from_jid: 'micahel.cross@example.com',
+    sent_date: '2017-06-05T20:08:38.263Z',
+    body_string: 'I meant thick',
+    body_text: ''
+  }, {
+    msg_id: '3334',
+    to_jid: 'uat-appsupport@conference.example.com',
+    from_jid: 'micahel.cross@example.com',
+    sent_date: '2017-06-05T20:08:38.263Z',
+    body_string: 'that is when I came on again',
+    body_text: ''
+  }],
+
+  userNotFound: [{
+    msg_id: '3334',
+    to_jid: 'uat-appsupport@conference.example.com',
+    from_jid: 'terrence.flynn@example.com',
+    sent_date: '2017-06-05T20:08:38.263Z',
+    body_string: 'that is when I came on again',
+    body_text: ''
+  }, {
+    msg_id: '3315',
+    to_jid: 'uat-appsupport@conference.example.com',
+    from_jid: 'micahel.cross@example.com',
+    sent_date: '2017-06-05T20:08:38.263Z',
+    body_string: 'I meant thick',
+    body_text: ''
+  }],
+
+  bodyNotFound: [{
+    msg_id: '3315',
+    to_jid: 'uat-appsupport@conference.example.com',
+    from_jid: 'micahel.cross@example.com',
+    sent_date: '2017-06-05T20:08:38.263Z',
+    body_string: '',
+    body_text: ''
+  }]
+}

lib/modules/test/fixtures/users.js

@@ -0,0 +1,57 @@
+module.exports = [
+  {
+    room_jid: 'admin@conference.example.com',
+    real_jid: 'micahel.cross@example.com'
+  }, {
+    room_jid: 'admin@conference.example.com',
+    real_jid: 'sbarclay@example.com'
+  }, {
+    room_jid: 'uat-appsupport@conference.example.com',
+    real_jid: 'anthony.brown@example.com'
+  }, {
+    room_jid: 'uat-appsupport@conference.example.com',
+    real_jid: 'james.seegar1@example.com'
+  }, {
+    room_jid: 'uat-appsupport@conference.example.com',
+    real_jid: 'jarrett.jennings2@example.com'
+  }, {
+    room_jid: 'uat-appsupport@conference.example.com',
+    real_jid: 'micahel.cross@example.com'
+  }, {
+    room_jid: 'uat-appsupport@conference.example.com',
+    real_jid: 'michael.rhoades3@example.com'
+  }, {
+    room_jid: 'uat-appsupport@conference.example.com',
+    real_jid: 'michael.rock@example.com'
+  }, {
+    room_jid: 'uat-appsupport@conference.example.com',
+    real_jid: 'terrence.flynn@example.com'
+  }, {
+    room_jid: 'uat-appsupport@conference.example.com',
+    real_jid: 'tony.dilisio2@example.com'
+  }, {
+    room_jid: 'uat-appsupport@conference.example.com',
+    real_jid: 'william.fleming2@example.com'
+  }, {
+    room_jid: 'test\\20room@conference.example.com',
+    real_jid: 'lt.u755@example.com'
+  }, {
+    room_jid: 'test\\20room@conference.example.com',
+    real_jid: 'lt.u755@example.com/9fyg1zayoi'
+  }, {
+    room_jid: 'the\\20cool\\20room@conference.example.com',
+    real_jid: 'cody.webb@example.com'
+  }, {
+    room_jid: 'tonyd_20161206_1432@conference.example.com',
+    real_jid: 'tony.dilisio2@example.com'
+  }, {
+    room_jid: 'tonyd_20161206_1432@conference.example.com',
+    real_jid: 'julie..sokol@example.com'
+  }, {
+    real_jid: 'mbcross',
+    room_jid: null
+  },{
+    real_jid: 'cross,\\20michael@chat.dhs.gov',
+    room_jid: null
+  }
+]

lib/modules/test/posts.js

@@ -0,0 +1,131 @@
+const expect = require('chai').expect
+const posts = require('../posts')
+const Fixtures = require('./fixtures')
+const FakeDB = require('./fakedb')
+const sink = require('./sink')
+const context = require('./context')()
+
+describe('modules.posts', function() {
+
+  before(function() {
+    //
+    // Set up context values
+    //
+    context.values = {
+      team: context.config.define.team,
+      channels: {
+        'uat-appsupport@conference.example.com': {
+          team: 'test',
+          name: 'uat-appsupport',
+          display_name: 'Uat Appsupport',
+          header: '',
+          purpose: '',
+          type: 'P'
+        }
+      },
+      users: {
+        'micahel.cross@example.com': {
+          username: 'micahel.cross',
+          email: 'micahel.cross@example.com',
+          auth_service: 'ldap',
+          teams: [
+            {
+              name: 'test',
+              channels: [
+                {
+                  name: 'uat-appsupport'
+                }
+              ]
+            }
+          ]
+        }
+      }
+    }
+  })
+
+  beforeEach(function() {
+    //
+    // Stub the output stream
+    //
+    context.output = sink()
+  })
+
+  it('should process post objects', function(done) {
+    //
+    // Set up the DB
+    //
+    context.jabber = new FakeDB(Fixtures.posts.ok)
+    //
+    // Process the posts
+    //
+    posts(context).then(function(c) {
+      expect(c).to.equal(context)
+      expect(context.output.write.callCount).to.equal(2)
+
+      let post = c.output.write.args[0][0]
+      expect(post).to.deep.equal({
+        type: 'post',
+        post: {
+          team: 'test',
+          channel: 'uat-appsupport',
+          user: 'micahel.cross',
+          message: 'I meant thick',
+          create_at: 1496693318263
+        }
+      })
+      expect(new Date(post.post.create_at).toISOString()).to.equal('2017-06-05T20:08:38.263Z')
+      done()
+    }).catch(function(e){
+      expect(e).to.be.undefined
+    })
+  })
+
+
+  it('should fail on user not found', function(done) {
+    //
+    // Set up the DB
+    //
+    context.jabber = new FakeDB(Fixtures.posts.userNotFound)
+    //
+    // Process the posts
+    //
+    posts(context).then(function(c) {
+      expect(c).to.equal(context)
+      expect(c.output.write.callCount).to.equal(1)
+      let post = c.output.write.args[0][0]
+      expect(post).to.deep.equal({
+        type: 'post',
+        post: {
+          team: 'test',
+          channel: 'uat-appsupport',
+          user: 'micahel.cross',
+          message: 'I meant thick',
+          create_at: 1496693318263
+        }
+      })
+      done()
+    }).catch(function(e){
+      expect(e).to.be.null
+      done()
+    })
+  })
+
+
+  it('should fail on body not found', function(done) {
+    //
+    // Set up the DB
+    //
+    context.jabber = new FakeDB(Fixtures.posts.bodyNotFound)
+    //
+    // Process the posts
+    //
+    posts(context).then(function(c) {
+      expect(c).to.equal(context)
+      expect(c.output.write.callCount).to.equal(0)
+      done()
+    }).catch(function(e){
+      expect(e).to.be.null
+      done()
+    })
+  })
+})

lib/modules/test/sink.js

@@ -0,0 +1,18 @@
+const { spy } = require('sinon')
+const { Writable } = require('stream')
+
+//
+// Returns a spied writable stream
+//
+module.exports = function() {
+
+  var writable = new Writable({
+    objectMode: true,
+    write(chunk, encoding, callback) {
+      return callback()
+    }
+  })
+  spy(writable, 'write')
+
+  return writable
+}

lib/modules/test/start.js

@@ -0,0 +1,30 @@
+const expect = require('chai').expect
+const fs = require('fs')
+const start = require('../start')
+const context = require('./context')()
+
+describe('modules.start', function() {
+  var resultHandler = function(err) { 
+    if(err) {
+      console.log('unlink failed', err)
+    } else {
+      console.log('file deleted')
+    }
+  }
+  
+  it('should set up the source and output', function(done) {
+    start(context)
+      .then(function (c) {
+        expect(c).to.equal(context)
+        expect(fs.existsSync(context.config.target.filename)).to.be.true
+        done()
+      })
+      .catch(function(err){
+        console.log(err)
+      })
+  })
+
+  afterEach(function() {
+    fs.unlink(context.config.target.filename, resultHandler)
+  })
+})

lib/modules/test/team.js

@@ -0,0 +1,27 @@
+const expect = require('chai').expect
+const team = require('../team')
+const context = require('./context')()
+
+describe('modules.team', function() {
+  it('should write a team object', function(done) {
+    team(context)
+      .then(function (c) {
+        expect(c).to.equal(context)
+        expect(c.output.write.args[0][0]).to.deep.equal({
+          type: 'team',
+          team: {
+            name: 'test',
+            display_name: 'Test Team',
+            description: 'Our Test Team',
+            type: 'I',
+            allow_open_invite: false
+          }
+        })
+        done()
+      })
+  })
+
+  afterEach(function() {
+    context.output.write.reset()
+  })
+})

lib/modules/test/users.js

@@ -0,0 +1,88 @@
+const expect = require('chai').expect
+const users = require('../users')
+const Fixtures = require('./fixtures')
+const context = require('./context')()
+
+describe('modules.users', function() {
+
+  before(function(){
+    context.values = {
+      team: context.config.define.team,
+      channels: {
+        'admin@conference.example.com': {
+          team: 'test',
+          name: 'admin',
+          display_name: 'Admin',
+          header: 'Admin Test room',
+          purpose: 'Admin Test room',
+          type: 'O'
+        },
+        'uat-appsupport@conference.example.com': {
+          team: 'hsin',
+          name: 'uat-appsupport',
+          display_name: 'Uat Appsupport',
+          header: '',
+          purpose: '',
+          type: 'P'
+        }
+      }
+    }
+  })
+
+  it('should process user objects', function(done) {
+
+    context.jabber.fetch.returns(Promise.resolve({recordset: Fixtures.users}))
+
+    users(context).then(function(c) {
+      expect(c).to.equal(context)
+      expect(Object.keys(c.values.users).length).equals(13)
+      expect(c.output.write.args[0][0]).to.deep.equal({
+        type: 'user',
+        user: {
+          username: 'micahel.cross',
+          email: 'micahel.cross@example.com',
+          auth_service: 'ldap',
+          auth_data: 'MICAHEL.CROSS',
+          teams: [
+            {
+              name: 'test',
+              channels: [
+                {
+                  name: 'admin'
+                }, {
+                  name: 'uat-appsupport'
+                }
+              ]
+            }
+          ]
+        }
+      })
+      expect(c.output.write.args[1][0]).to.deep.equal({
+        type: 'user',
+        user: {
+          username: 'sbarclay',
+          email: 'sbarclay@example.com',
+          auth_service: 'ldap',
+          auth_data: 'SBARCLAY',
+          teams: [
+            {
+              name: 'test',
+              channels: [
+                {
+                  name: 'admin'
+                }
+              ]
+            }
+          ]
+        }
+      })
+
+      done()
+    })
+  })
+
+  afterEach(function() {
+    context.jabber.fetch.reset()
+    context.output.write.reset()
+  })
+})

lib/modules/test/version.js

@@ -0,0 +1,21 @@
+const expect = require('chai').expect
+const version = require('../version')
+const context = require('./context')()
+
+describe('modules.version', function() {
+  it('should write a version object', function(done) {
+    version(context)
+      .then(function (c) {
+        expect(c).to.equal(context)
+        expect(c.output.write.args[0][0]).to.deep.equal({
+          type: 'version',
+          version: 1
+        })
+        done()
+      })
+  })
+
+  afterEach(function() {
+    context.output.write.reset()
+  })
+})

lib/modules/transform.js

@@ -0,0 +1,9 @@
+const { Transform } = require('stream')
+
+module.exports = function(transform, callback) {
+  return new Transform({
+    readableObjectMode: true,
+    writableObjectMode: true,
+    transform
+  }).on('finish', callback)
+}

lib/modules/users.js

@@ -0,0 +1,115 @@
+const _ = require('lodash')
+const Factory = require('../factory')
+const Utils = require('./utils')
+
+//
+// Initialize the child logger for
+// the module
+//
+const log = require('../log').child({
+  module: 'users'
+})
+
+module.exports = function(context) {
+  //
+  // Select all of the users
+  //
+  const where = 'msg_type = \'c\''
+
+  return context.jabber.fetch(`
+      SELECT real_jid, room_jid FROM dbo.tc_users
+      UNION ALL (
+        SELECT from_jid AS real_jid, NULL AS room_jid FROM dbo.jm WHERE ${where}
+        UNION
+        SELECT to_jid AS real_jid, NULL AS room_jid FROM dbo.jm WHERE ${where}
+      )
+    `)
+    //
+    // Build up the user objects and then
+    // write them to the output
+    //
+    .then(function(results) {
+      log.info(`${results.recordset.length} records found`)
+      //
+      // Map of users
+      //
+      var users = {}
+      //
+      // Iterate over the record set and
+      // assemble the user objects
+      //
+      results.recordset.forEach(function(record) {
+        log.debug(record)
+        //
+        // Clean the real_jid to ensure we don't
+        // have duplicates with /<string> suffixes
+        //
+        var real_jid = Utils.realJID(record.real_jid)
+        //
+        // Generate the username fro the real_jid
+        //
+        var username = toUsername(real_jid)
+        //
+        // Return a reference to the user in the user map
+        // or add one if it doesn't yet exist
+        //
+        var user = users[real_jid] = _.get(users, real_jid, {
+          username,
+          email: real_jid,
+          auth_service: context.config.define.user.auth_service,
+          auth_data: username.toUpperCase(),
+          teams: [{
+            name: context.values.team.name,
+            channels: []
+          }]
+        })
+        //
+        // Look up the channel based on the
+        // room id. For direct messages, the room_jid
+        // will be null.
+        //
+        var channel = context.values.channels[record.room_jid]
+        //
+        // Add it to the user
+        //
+        if (channel) {
+          user.teams[0].channels = _.unionBy(user.teams[0].channels, [{
+            name: channel.name
+          }], 'name')
+        } else {
+          record.room_jid && log.warn(`... channel not found for ${record.room_jid}`)
+        }
+      })
+      //
+      // Now that the users are assembled, write
+      // them to the output
+      //
+      _.forEach(users, function(user, key) {
+        try {
+          context.output.write(
+            Factory.user(user)
+          )
+          log.info(`... writing ${user.username}`)
+        }
+        catch(err) {
+          log.error(`... ignoring ${user.username} on error: ${err.message}.`)
+          delete users[key]
+        }
+      })
+      //
+      // Add the users map to the context
+      //
+      context.values.users = users
+      //
+      // Return the context
+      //
+      return context
+    })
+}
+
+//
+// Parse the username
+//
+const toUsername = function (jid='') {
+  return jid.split('@')[0]
+}

lib/modules/utils.js

@@ -0,0 +1,103 @@
+const _ = require('lodash')
+
+//
+// Declare utils object
+//
+const utils = {}
+
+//
+//
+//
+utils.chunk = function(body) {
+  //
+  // Use regex to create an array
+  // of strings of max length
+  //
+  return body.match(/[\s\S]{1,4000}/g)
+}
+
+//
+// Removes trailing /<string> suffixes that
+// may exist in the user ids. This happens
+// if a user logs in to jabber more than
+// once at the same time
+//
+utils.realJID = function (jid='') {
+  return jid.split('/')[0]
+    .replace(/\\20+/g, '.')
+    .replace(/\.\.+/, '.')
+}
+
+//
+// Lookup values
+//
+utils.lookup = function (type, map, key) {
+  var found = map[key]
+
+  if(!found) {
+    throw new Error(`${type} ${key} not found`)
+  }
+
+  return found
+}
+
+//
+// Obtain the username from a jid
+//
+utils.username = function (users, jid) {
+  return utils.lookup('user', users, utils.realJID(jid)).username
+}
+
+//
+// Obtain the channel name from a jid
+//
+utils.channelName = function (channels, jid) {
+  return utils.lookup('channel', channels, jid).name
+}
+
+//
+// Find the message body
+//
+utils.body = function (message) {
+  var body = message.body_string || message.body_text
+
+  if(!body) {
+    throw new Error(`message ${message.msg_id} body is empty`)
+  }
+
+  return utils.chunk(body)
+}
+
+//
+// Coverts the to / from JIDs to a members
+// array
+//
+utils.members = function(users, to, from) {
+  //
+  // We use uniq to remove duplicate
+  // members in the same channel
+  //
+  return _.uniq(_.sortBy([
+    utils.username(users, to),
+    utils.username(users, from),
+  ]))
+}
+
+//
+// Checks if the members list is valid
+//
+utils.membersAreValid = function(members) {
+  return _.isArray(members) && members.length > 1
+}
+
+//
+// Convert ISO to millis
+//
+utils.millis = function(date) {
+  return new Date(date).getTime()
+}
+
+//
+// Export the functions
+//
+module.exports = utils

lib/modules/version.js

@@ -0,0 +1,12 @@
+const Factory = require('../factory')
+
+module.exports = function(context) {
+  //
+  // Write the version object
+  //
+  context.output.write(Factory.version())
+  //
+  // Return a resolved promise
+  //
+  return Promise.resolve(context)
+}

lib/mongo.js

@@ -0,0 +1,28 @@
+const mongo = require('mongodb')
+
+//
+// Declare internals
+//
+let client = {}
+
+//
+// Connect to the db and create a connection
+// pool
+//
+module.exports.connect = function (config) {
+  client = new mongo.MongoClient(config.source.uri)
+  return client.connect()
+}
+
+module.exports.close = function () {
+  return client.close()
+}
+
+module.exports.collection = function (collection) {
+  return client.db().collection(collection)
+}
+
+module.exports.gridFsBucket = function (bucketName) {
+  return new mongo.GridFSBucket(client.db(), { bucketName })
+}
+

lib/mssql.js

@@ -0,0 +1,57 @@
+const SQL = require('mssql')
+const log = require('./log')
+
+//
+// Declare internals
+//
+const internals = {}
+
+//
+// Connect to the db and create a connection
+// pool
+//
+module.exports.connect = function(config) {
+  return SQL.connect(config.source.uri)
+    .then(function(pool) {
+      internals.pool = pool
+    })
+}
+
+//
+// Returns all results in one batch
+//
+module.exports.fetch = function(query) {
+  return internals.pool.request().query(query)
+}
+
+//
+// Delivers results to the specified write
+// stream. Use this for large result sets.
+//
+module.exports.pipe = function(query, writable) {
+  //
+  // Create the request
+  //
+  const request = internals.pool.request()
+  //
+  // Attach the writable stream
+  //
+  request.pipe(writable)
+  //
+  // Submit the query
+  //
+  request.query(query)
+  //
+  // Return the writable stream for
+  // chaining
+  //
+  return writable
+}
+
+//
+// Set up an error handler
+//
+SQL.on('error', function(err) {
+  log.error(err)
+  throw(err)
+})

lib/rocketchat/channels.js

@@ -0,0 +1,55 @@
+const _ = require('lodash')
+const Factory = require('../factory')
+const slug = require("slug");
+
+//
+// Initialize the child logger for
+// the module
+//
+const log = require('../log').child({
+  module: 'channels'
+})
+
+module.exports = async function (context) {
+  const collection = context.rocketchat.roomsCollection()
+  const query = { t: { $in: ['c', 'p'] } }
+
+  let channels = {}
+  const cursor = collection.find(query)
+  const mergeDiscussions = _.get(context, 'config.define.channels.mergeDiscussionIntoParent', false)
+  const discussions = {}
+  while (await cursor.hasNext()) {
+    const result = await cursor.next()
+    if (mergeDiscussions && result.prid) {
+      discussions[result._id] = result.prid
+      log.info(`... skipping channel ${result._id} to merge discussion into parent ${result.prid}`)
+      continue;
+    }
+
+    // Define the channel and add it to the context
+    let channel = channels[result._id] = {
+      team: context.values.team.name,
+      name: slug(_.toLower(result.name)),
+      display_name: result.name,
+      header: result.topic,
+      purpose: result.description,
+      type: result.t === 'p' ? 'P' : 'O'
+    }
+    const map = _.get(context, `config.define.channels.map.${result.name}`, false)
+    if (map) {
+      channel.name = map.name
+      channel.display_name = map.display_name
+    }
+    // Write the channel data to the output
+    log.info(`... writing ${channel.name} (${result.name})`)
+    context.output.write(
+      Factory.channel(channel)
+    )
+  }
+  if (!context.values.discussions) {
+    context.values.discussions = {}
+  }
+  context.values.discussions = Object.assign(context.values.discussions, discussions)
+  context.values.channels = channels
+  return context
+}

lib/rocketchat/directChannels.js

@@ -0,0 +1,60 @@
+const Factory = require('../factory')
+const Utils = require('./utils')
+const _ = require("lodash");
+
+//
+// Initialize the child logger for
+// the module
+//
+const log = require('../log').child({
+  module: 'directChannels'
+})
+
+module.exports = async function(context) {
+  // Select all of the rooms
+  const collection = context.rocketchat.roomsCollection()
+  const query = { t: 'd' }
+
+  const directChannels = {}
+  const cursor = collection.find(query)
+  const mergeDiscussions = _.get(context, 'config.define.channels.mergeDiscussionIntoParent', false)
+  const discussions = {}
+  while (await cursor.hasNext()) {
+    const result = await cursor.next()
+    if (mergeDiscussions && result.prid) {
+      discussions[result._id] = result.prid
+      log.info(`... skipping direct channel ${result._id} to merge discussion into parent ${result.prid}`)
+      continue;
+    }
+
+    try {
+      // Generate the members array for the direct channel
+      let members = Utils.members(
+        context.values.users,
+        result.uids,
+      )
+      if (members.length === 1) {
+        members.push(members[0])
+      }
+      // If there at least two members
+      if (Utils.membersAreValid(members)) {
+        log.info(`... writing members (${members.join(', ')})`)
+        let channel = directChannels[result._id] = {
+          header: result.topic,
+          members
+        }
+        context.output.write(
+          Factory.directChannel(channel)
+        )
+      }
+    } catch (err) {
+      log.error(`... ignoring directChannel members (${result.usernames.join(', ')}) on error: ${err.message}.`)
+    }
+  }
+  if (!context.values.discussions) {
+    context.values.discussions = {}
+  }
+  context.values.discussions = Object.assign(context.values.discussions, discussions)
+  context.values.directChannels = directChannels
+  return context
+}

lib/rocketchat/emoji.js

@@ -0,0 +1,75 @@
+const Factory = require('../factory')
+const Utils = require('./utils')
+
+//
+// Initialize the child logger for
+// the module
+//
+const log = require('../log').child({
+  module: 'custom emoji'
+})
+
+module.exports = async function (context) {
+  const collection = context.rocketchat.emojiCollection()
+  const fileCollection = context.rocketchat.collection('custom_emoji')
+
+  const cursor = collection.find()
+  while (await cursor.hasNext()) {
+    const result = await cursor.next()
+
+    // Prepare destination file
+    const filename = `${result.name}.${result.extension}`
+    const dest = `${context.config.target.filesPath}/custom_emoji/${filename}`
+    // Download emoji file
+    if (result.store.startsWith('FileSystem:')) {
+      const src = context.config.source.customEmojiPath
+      const srcFilename = Utils.srcPath(src, filename)
+      if (!srcFilename) {
+        return new Error(`source file "${filename}" not found`)
+      }
+      // Copy to output dir
+      Utils.copyFile(srcFilename, dest)
+    } else if (result.store.startsWith('GridFS:')) {
+      await Utils.downloadGridFS(context, fileCollection, filename, dest)
+    } else {
+      throw new Error(`file system ${file.store} is not supported. Migrate to FileSystem first, see readme.`)
+    }
+
+    // Export custom emoji
+    let emoji = {
+      name: result.name,
+      image: dest,
+    }
+    log.info(`... writing ${emoji.name}`)
+    context.output.write(
+      Factory.emoji(emoji)
+    )
+  }
+  return context
+}
+
+const exportFile = async function (context, collection, fileId) {
+  const type = collection.collectionName
+  const file = await collection.findOne({ _id: fileId })
+  if (!file) {
+    throw new Error(`file ${fileId} from collection ${type} is missing`)
+  }
+  const dest = context.config.target.filesPath
+  const destFilename = utils.destPath(dest, collection.collectionName, file)
+
+  if (file.store.startsWith('FileSystem:')) {
+    const src = context.config.source.uploadsPath
+    const srcFilename = utils.srcPath(src, file._id)
+    if (!srcFilename) {
+      return new Error(`source file "${file._id}" not found`)
+    }
+    // Copy to output dir
+    utils.copyFile(srcFilename, destFilename)
+  } else if (file.store.startsWith('GridFS:')) {
+    await utils.downloadGridFS(context, collection, file._id, destFilename)
+  } else {
+    throw new Error(`file system ${file.store} is not supported. Migrate to FileSystem first, see readme.`)
+  }
+
+  return destFilename
+}

lib/rocketchat/end.js

@@ -0,0 +1,13 @@
+//
+// Initialize the child logger for
+// the module
+//
+const log = require('../log').child({
+  module: 'end'
+})
+
+module.exports = function(context) {
+  log.info('end')
+  context.rocketchat.close()
+  context.output.end()
+}

lib/rocketchat/index.js

@@ -0,0 +1,21 @@
+const start = require('./start')
+const version = require('./version')
+const emoji = require('./emoji')
+const team = require('./team')
+const channels = require('./channels')
+const users = require('./users')
+const posts = require('./posts')
+const directChannels = require('./directChannels')
+const end = require('./end')
+
+module.exports = {
+  start,
+  version,
+  emoji,
+  team,
+  channels,
+  users,
+  posts,
+  directChannels,
+  end
+}

lib/rocketchat/posts.js

@@ -0,0 +1,162 @@
+const Factory = require('../factory')
+const Utils = require('./utils')
+const _ = require('lodash')
+
+//
+// Initialize the child logger for
+// the module
+//
+const log = require('../log').child({
+  module: 'posts'
+})
+
+module.exports = async function (context) {
+  const collection = context.rocketchat.messagesCollection()
+  // Keep track of the number of posts written for logging
+  let written = 0
+
+  let memReplyIds = {}
+  const total = await collection.count()
+  const cursor = collection.find()
+  while (await cursor.hasNext()) {
+    const result = await cursor.next()
+    let posts = await collectPostData(context, result, memReplyIds, false)
+    posts.forEach(function(post) {
+      context.output.write(post.isDirect ? Factory.directPost(post) : Factory.post(post))
+    })
+    // Log progress periodically
+    written += posts.length
+    if (written % 1000 == 0) {
+      log.info(`... wrote ${written} posts`)
+    }
+  }
+  log.info(`... finished exporting ${written} posts, ignored ${total - written}`)
+  return context
+}
+
+async function collectPostData(context, result, memReplyIds, isReply) {
+  if (memReplyIds.hasOwnProperty(result._id)) {
+    return []
+  }
+
+  try {
+    // Try to get discussion channel id
+    let channelId = result.rid
+    const mergeDiscussions = _.get(context, 'config.define.channels.mergeDiscussionIntoParent', false)
+    const parentId = _.get(context, `values.discussions.${channelId}`, false)
+    if (mergeDiscussions && parentId) {
+      channelId = parentId
+    }
+    // Check if direct message
+    let isDirect = !context.values.channels[channelId] && !!context.values.directChannels[channelId]
+    let post = {}
+    let channelInfo = {}
+
+    if (isDirect) {
+      let { members: channel_members } = context.values.directChannels[channelId]
+      // Ensure we have at least two channel members before we can write the message
+      if (Utils.membersAreValid(channel_members)) {
+        channelInfo = { channel_members }
+      } else {
+        log.error(`... ignoring message id:${result._id} on error: directChannel ${channelId} not found.`)
+        return []
+      }
+    } else {
+      channelInfo = {
+        team: context.values.team.name,
+        channel: Utils.channelName(
+          context.values.channels, channelId
+        ),
+      }
+    }
+    if (!isReply) {
+      Object.assign(post, channelInfo)
+    }
+
+    const reactions = Object.keys(result.reactions || {}).reduce((prev, code) => {
+      return result.reactions[code].usernames.map((u) => {
+        return {
+          user: u,
+          emoji_name: _.trim(code, ':'),
+          create_at: Utils.millis(result.ts),
+        }
+      }).concat(prev)
+    }, [])
+    const flagged_by = (result.starred || []).map(({ _id: uid }) => {
+      try {
+        return Utils.username(context.values.users, uid)
+      } catch (err) {
+        return undefined
+      }
+    }).filter(v => v)
+
+    Object.assign(post, {
+      user: Utils.username(
+        context.values.users, result.u._id
+      ),
+      create_at: Utils.millis(result.ts),
+      reactions,
+      flagged_by,
+      isDirect,
+    })
+
+    // Collect data from attachments
+    let attachments = Utils.processAttachments(context, result)
+    let file
+    let body = result.msg
+
+    await Promise.all(attachments.map(async (a) => {
+      if (a.type === 'file') {
+        file = await a.data
+      } else if (a.type === 'quote') {
+        body = a.data
+      }
+    }))
+    if (file && file instanceof Error) {
+      throw file
+    }
+
+    if (file && file.description) {
+      body = `File description: \n${file.description} \n\n ${body}`
+    }
+    const chunks = body ? Utils.body(body) : [body]
+    let posts = chunks.map((chunk) => {
+      return Object.assign({}, post, {
+        message: chunk
+      })
+    })
+
+    if (!posts[0]) {
+      throw new Error(`Post is empty`)
+    }
+
+    if (file) {
+      posts[0].attachments = [{
+        path: file.path
+      }]
+    }
+
+    const replies = []
+    if (!isReply) {
+      const collection = context.rocketchat.messagesCollection()
+      const cursor = collection.find({tmid: result._id })
+      while (await cursor.hasNext()) {
+        const reply = await cursor.next()
+        const replyData = await collectPostData(context, reply, memReplyIds, true)
+        replyData.forEach(r => replies.push(r))
+        memReplyIds[reply._id] = true
+      }
+    }
+
+    Object.assign(posts[0], {
+      reactions,
+      flagged_by,
+      replies,
+    })
+
+    return posts
+  } catch (err) {
+    log.error(`... ignoring message id:${result._id} on error: ${err.message}.`)
+    return []
+  }
+}

lib/rocketchat/start.js

@@ -0,0 +1,32 @@
+const datafile = require('../datafile')
+const fs = require('fs');
+
+//
+// Initialize the child logger for
+// the module
+//
+const log = require('../log').child({
+  module: 'start'
+})
+
+module.exports = async function(context) {
+  log.info('preparing files paths')
+  const dest = context.config.target.filesPath
+  if (!dest || !fs.existsSync(dest)) {
+    fs.mkdirSync(dest)
+  }
+  if (!fs.lstatSync(dest).isDirectory() || fs.accessSync(dest, fs.constants.W_OK)) {
+    throw new Error(`Directory "${dest} is not writable"`)
+  }
+
+  log.info('connecting to rocketchat')
+  await context.rocketchat.connect(context.config)
+  log.info(`creating file '${context.config.target.filename}'`)
+  // Create the datafile and add it to the context
+  context.output = datafile(
+    context.config.target.filename,
+    process.exit
+  )
+
+  return context
+}

lib/rocketchat/team.js

@@ -0,0 +1,29 @@
+const Factory = require('../factory')
+
+//
+// Initialize the child logger for
+// the module
+//
+const log = require('../log').child({
+  module: 'team'
+})
+
+module.exports = function(context) {
+  log.info(`writing team '${context.config.define.team.name}'`)
+  //
+  // Store the team object for use
+  // by other modules
+  //
+  context.values.team = context.config.define.team
+  //
+  // Write the team object to the
+  // output
+  //
+  context.output.write(Factory.team(
+    context.config.define.team
+  ))
+  //
+  // Return a resolved promise
+  //
+  return Promise.resolve(context)
+}

lib/rocketchat/users.js

@@ -0,0 +1,105 @@
+const Factory = require('../factory')
+const Utils = require('./utils')
+const { Gitlab } = require('@gitbeaker/node')
+const _ = require('lodash')
+
+//
+// Initialize the child logger for
+// the module
+//
+const log = require('../log').child({
+  module: 'users'
+})
+
+module.exports = async function(context) {
+  const collection = context.rocketchat.usersCollection()
+
+  let users = {}
+  const cursor = collection.find({ type: 'user' })
+  while (await cursor.hasNext()) {
+    const result = await cursor.next()
+    const id = result._id
+    const [first_name, last_name] = result.name.split(' ')
+    try {
+      const ldapAuthService = _.get(context, 'config.define.user.ldap_auth_service', '')
+      const roleMap = _.get(context, 'config.define.user.globalRoleMap', {})
+      const auth_service = result.ldap ? ldapAuthService : ''
+      if (result.roles.indexOf('admin') !== -1 && result.roles.indexOf('user') === -1) {
+        result.roles.push('user')
+      }
+      const highlights = _.get(result, 'settings.preferences.highlights', [])
+      let user = users[id] = {
+        username: result.username,
+        first_name,
+        last_name,
+        auth_service,
+        auth_data: await getAuthData(context, result, auth_service),
+        email: result.emails[0].address,
+        roles: result.roles.sort().reduce((roles, role) => {
+          if (roleMap.hasOwnProperty(role)) {
+            roles.push(roleMap[role])
+          }
+          return roles
+        }, []).join(' '),
+        teams: [{
+          name: context.values.team.name,
+          channels: result.__rooms.map((id) => {
+            const name = _.get(context, `values.channels.${id}.name`, false)
+            return name ? { name } : undefined
+          }).filter(v => v)
+        }],
+        notify_props: {
+          mention_keys: highlights.map(s => s.replace('@', '')).join(','),
+        },
+      }
+      if (auth_service) {
+        user.auth_service = auth_service
+        user.auth_data = await getAuthData(context, result, auth_service)
+      }
+      if (!user.roles.length) {
+        delete user.roles
+      }
+      const avatar = await getAvatar(context, result)
+      if (avatar) {
+        user.profile_image = avatar
+      }
+
+      context.output.write(
+        Factory.user(user)
+      )
+      log.info(`... writing ${user.username}`)
+    }
+    catch(err) {
+      log.error(`... ignoring ${result.username} on error: ${err.message}.`)
+      delete users[id]
+    }
+  }
+  context.values.users = users
+  return context
+}
+
+async function getAuthData(context, user, auth_service) {
+  switch (auth_service) {
+    case 'gitlab':
+      const {host, token} = _.get(context, 'config.define.user.gitlab')
+      const api = new Gitlab({ host, token });
+      const gitlabUser = await api.Users.username(user.username);
+      if (!gitlabUser || !gitlabUser.length) {
+        throw new Error(`user ${user.username} is missing in Gitlab`)
+      }
+      return `${gitlabUser[0].id}`
+    case 'ldap':
+      return user.username.toUpperCase()
+    default:
+      return ''
+  }
+}
+
+async function getAvatar(context, user) {
+  const collection = context.rocketchat.avatarsCollection()
+  const avatar = await collection.findOne({userId: user._id}, {sort: {_updatedAt: -1}})
+  if (!avatar) {
+    return ""
+  }
+  return Utils.exportFile(context, collection, avatar._id)
+}

lib/rocketchat/utils.js

@@ -0,0 +1,191 @@
+const _ = require('lodash')
+const fs = require('fs')
+const path = require('path')
+
+//
+// Declare utils object
+//
+const utils = {}
+
+//
+//
+//
+utils.chunk = function(body) {
+  //
+  // Use regex to create an array
+  // of strings of max length
+  //
+  return body.match(/[\s\S]{1,4000}/g)
+}
+
+//
+// Lookup values
+//
+utils.lookup = function (type, map, key) {
+  var found = map[key]
+
+  if(!found) {
+    throw new Error(`${type} ${key} not found`)
+  }
+
+  return found
+}
+
+//
+// Obtain the username from a jid
+//
+utils.username = function (users, id) {
+  return utils.lookup('user', users, id).username
+}
+
+//
+// Obtain the channel name from a jid
+//
+utils.channelName = function (channels, id) {
+  return utils.lookup('channel', channels, id).name
+}
+
+//
+// Find the message body
+//
+utils.body = function (body) {
+  return utils.chunk(body)
+}
+
+utils.processAttachments = function (context, message) {
+  if (!message.attachments) {
+    return []
+  }
+  return message.attachments.map((attachment) => {
+    if (attachment.type && attachment.type === 'file') {
+      return {
+        type: 'file',
+        data: utils.processFileAttachment(context, message, attachment),
+      }
+    }
+    if (attachment.text) {
+      return {
+        type: 'quote',
+        data: utils.processQuoteAttachment(message, attachment)
+      }
+    }
+    return {
+      type: 'unknown'
+    }
+  })
+}
+
+utils.processQuoteAttachment = function (message, attachment) {
+  if (!attachment.text) {
+    return ''
+  }
+  const body = (message.msg || '').replace(/\[ \]\(.*msg=.*?\) /, '')
+  // Convert quote to markdown
+  return `@${attachment.author_name}:\n ${attachment.text.replace(/^/, '> ')} \n\n${body}`
+}
+
+utils.processFileAttachment = async function (context, message, attachment) {
+  if (!message.file) {
+    return new Error(`message ${message._id} file is missing`)
+  }
+  const collection = context.rocketchat.uploadsCollection()
+  return {
+    description: attachment.description,
+    path: await utils.exportFile(context, collection, message.file._id),
+  }
+}
+
+utils.srcPath = function (srcDir, filename) {
+  const files = fs.readdirSync(srcDir).filter((fn) => fn.startsWith(filename));
+  if (files.length !== 1) {
+    return false
+  }
+  return `${srcDir}/${path.basename(files[0])}`
+}
+
+utils.destPath = function (destDir, type, file) {
+  let dest = `${destDir}/${type}/${file._id}/${file.name}`;
+  const ext = path.extname(dest)
+  if (!ext || !/^\.[A-Za-z][A-Za-z0-9]*$/.test(ext)) {
+    if (file.identify && file.identify.format) {
+      dest += '.' + file.identify.format
+    } else {
+      dest += '.' + file.type.split('/')[1]
+    }
+  }
+  return dest
+}
+
+utils.exportFile = async function (context, collection, fileId) {
+  const type = collection.collectionName
+  const file = await collection.findOne({ _id: fileId })
+  if (!file) {
+    throw new Error(`file ${fileId} from collection ${type} is missing`)
+  }
+  const dest = context.config.target.filesPath
+  const destFilename = utils.destPath(dest, collection.collectionName, file)
+
+  if (file.store.startsWith('FileSystem:')) {
+    const src = context.config.source.uploadsPath
+    const srcFilename = utils.srcPath(src, file._id)
+    if (!srcFilename) {
+      return new Error(`source file "${file._id}" not found`)
+    }
+    // Copy to output dir
+    utils.copyFile(srcFilename, destFilename)
+  } else if (file.store.startsWith('GridFS:')) {
+    await utils.downloadGridFS(context, collection, file._id, destFilename)
+  } else {
+    throw new Error(`file system ${file.store} is not supported. Migrate to FileSystem first, see readme.`)
+  }
+
+  return destFilename
+}
+
+utils.copyFile = function (src, dest) {
+  if (!fs.existsSync(path.dirname(dest))) {
+    fs.mkdirSync(path.dirname(dest), { recursive: true })
+  }
+  if (!fs.existsSync(dest)) {
+    fs.copyFileSync(src, dest)
+  }
+}
+
+utils.downloadGridFS = async function (context, collection, id, dest) {
+  if (!fs.existsSync(path.dirname(dest))) {
+    fs.mkdirSync(path.dirname(dest), { recursive: true })
+  }
+  if (fs.existsSync(dest)) {
+    fs.unlinkSync(dest)
+  }
+
+  const bucket = context.rocketchat.gridFsBucket(collection.collectionName)
+  const destStream = fs.createWriteStream(dest);
+  bucket.openDownloadStream(id).pipe(destStream);
+  return new Promise((resolve, reject) => {
+    destStream.on('finish', resolve);
+  })
+}
+
+utils.members = function(users, usernames) {
+  return _.uniq(_.sortBy(usernames.map((username) => utils.username(users, username))))
+}
+
+//
+// Checks if the members list is valid
+//
+utils.membersAreValid = function(members) {
+  return _.isArray(members) && members.length > 0
+}
+
+//
+// Convert ISO to millis
+//
+utils.millis = function(date) {
+  return new Date(date).getTime()
+}
+
+//
+// Export the functions
+//
+module.exports = utils

lib/rocketchat/version.js

@@ -0,0 +1,12 @@
+const Factory = require('../factory')
+
+module.exports = function(context) {
+  //
+  // Write the version object
+  //
+  context.output.write(Factory.version())
+  //
+  // Return a resolved promise
+  //
+  return Promise.resolve(context)
+}

lib/test/datafile.js

@@ -0,0 +1,33 @@
+const expect = require('chai').expect
+const fs = require('fs')
+const datafile = require('../datafile')
+
+const filename = 'test.json'
+
+describe('datafile', function() {
+
+  var resultHandler = function(err) { 
+    if(err) {
+      console.log('unlink failed', err)
+    } else {
+      console.log('file deleted')
+    }
+  }
+
+  it('should write objects to a file', function(done) {
+    var d = datafile(filename, () => {
+      expect(fs.existsSync(filename)).to.be.true
+      done()
+    })
+
+    d.write({
+      foo: 'bar'
+    })
+
+    d.end()
+  })
+
+  after(function() {
+    fs.unlink(filename, resultHandler)
+  })
+})

package.json

@@ -0,0 +1,42 @@
+{
+  "name": "loop-etl-rocketchat",
+  "version": "1.0.0",
+  "description": "An ETL framework to migrate data to Loop",
+  "author": "Michael DeBonis",
+  "license": "MIT",
+  "homepage": "https://git.wilix.dev/loop/loop-etl-rocketchat#readme",
+  "repository": {
+    "type": "git",
+    "url": "git+https://git.wilix.dev/loop/loop-etl-rocketchat.git"
+  },
+  "bugs": {
+    "url": "https://git.wilix.dev/loop/loop-etl-rocketchat/issues"
+  },
+  "dependencies": {
+    "@gitbeaker/node": "^35.6.0",
+    "bunyan": "^1.8.10",
+    "joi": "^10.6.0",
+    "lodash": "^4.17.4",
+    "mongodb": "^4.7.0",
+    "mssql": "^4.0.4",
+    "slug": "^0.9.1",
+    "xmldoc": "^1.1.0"
+  },
+  "devDependencies": {
+    "chai": "^4.0.2",
+    "codeclimate-test-reporter": "^0.5.0",
+    "eslint": "^4.1.0",
+    "istanbul": "^0.4.5",
+    "mocha": "^3.4.2",
+    "sinon": "^2.3.4"
+  },
+  "main": "index.js",
+  "scripts": {
+    "lint": "eslint *.js context lib --ext .js",
+    "test": "npm run lint && mocha lib --recursive",
+    "test-watch": "npm run test -- --watch",
+    "test-coverage": "istanbul cover _mocha lib -- --recursive",
+    "start": "node index.js | bunyan",
+    "start:rocketchat": "node rocketchat.js | bunyan"
+  }
+}

rocketchat.js

@@ -0,0 +1,46 @@
+const context = require('./context/rocketchat')
+const log = require('./lib/log')
+const {
+  start,
+  version,
+  emoji,
+  team,
+  channels,
+  users,
+  posts,
+  directChannels,
+  end
+} = require('./lib/rocketchat')
+
+//
+// Common function log errors and
+// terminate the process
+//
+const abort = function (err) {
+  log.error(err)
+  //
+  // We set a timeout here to
+  // allow the log streams to finish
+  // writing.
+  //
+  setTimeout(function () {
+    process.exit(1)
+  }, 3000)
+}
+
+//
+// Ensure we trap uncaught exceptions
+// and properly abort
+//
+process.on('uncaughtException', abort)
+
+start(context)
+  .then(version)
+  .then(emoji)
+  .then(team)
+  .then(channels)
+  .then(users)
+  .then(directChannels)
+  .then(posts)
+  .then(end)
+  .catch(abort)