init

lib/rocketchat/channels.js

@@ -0,0 +1,55 @@
+const _ = require('lodash')
+const Factory = require('../factory')
+const slug = require("slug");
+
+//
+// Initialize the child logger for
+// the module
+//
+const log = require('../log').child({
+  module: 'channels'
+})
+
+module.exports = async function (context) {
+  const collection = context.rocketchat.roomsCollection()
+  const query = { t: { $in: ['c', 'p'] } }
+
+  let channels = {}
+  const cursor = collection.find(query)
+  const mergeDiscussions = _.get(context, 'config.define.channels.mergeDiscussionIntoParent', false)
+  const discussions = {}
+  while (await cursor.hasNext()) {
+    const result = await cursor.next()
+    if (mergeDiscussions && result.prid) {
+      discussions[result._id] = result.prid
+      log.info(`... skipping channel ${result._id} to merge discussion into parent ${result.prid}`)
+      continue;
+    }
+
+    // Define the channel and add it to the context
+    let channel = channels[result._id] = {
+      team: context.values.team.name,
+      name: slug(_.toLower(result.name)),
+      display_name: result.name,
+      header: result.topic,
+      purpose: result.description,
+      type: result.t === 'p' ? 'P' : 'O'
+    }
+    const map = _.get(context, `config.define.channels.map.${result.name}`, false)
+    if (map) {
+      channel.name = map.name
+      channel.display_name = map.display_name
+    }
+    // Write the channel data to the output
+    log.info(`... writing ${channel.name} (${result.name})`)
+    context.output.write(
+      Factory.channel(channel)
+    )
+  }
+  if (!context.values.discussions) {
+    context.values.discussions = {}
+  }
+  context.values.discussions = Object.assign(context.values.discussions, discussions)
+  context.values.channels = channels
+  return context
+}

lib/rocketchat/directChannels.js

@@ -0,0 +1,60 @@
+const Factory = require('../factory')
+const Utils = require('./utils')
+const _ = require("lodash");
+
+//
+// Initialize the child logger for
+// the module
+//
+const log = require('../log').child({
+  module: 'directChannels'
+})
+
+module.exports = async function(context) {
+  // Select all of the rooms
+  const collection = context.rocketchat.roomsCollection()
+  const query = { t: 'd' }
+
+  const directChannels = {}
+  const cursor = collection.find(query)
+  const mergeDiscussions = _.get(context, 'config.define.channels.mergeDiscussionIntoParent', false)
+  const discussions = {}
+  while (await cursor.hasNext()) {
+    const result = await cursor.next()
+    if (mergeDiscussions && result.prid) {
+      discussions[result._id] = result.prid
+      log.info(`... skipping direct channel ${result._id} to merge discussion into parent ${result.prid}`)
+      continue;
+    }
+
+    try {
+      // Generate the members array for the direct channel
+      let members = Utils.members(
+        context.values.users,
+        result.uids,
+      )
+      if (members.length === 1) {
+        members.push(members[0])
+      }
+      // If there at least two members
+      if (Utils.membersAreValid(members)) {
+        log.info(`... writing members (${members.join(', ')})`)
+        let channel = directChannels[result._id] = {
+          header: result.topic,
+          members
+        }
+        context.output.write(
+          Factory.directChannel(channel)
+        )
+      }
+    } catch (err) {
+      log.error(`... ignoring directChannel members (${result.usernames.join(', ')}) on error: ${err.message}.`)
+    }
+  }
+  if (!context.values.discussions) {
+    context.values.discussions = {}
+  }
+  context.values.discussions = Object.assign(context.values.discussions, discussions)
+  context.values.directChannels = directChannels
+  return context
+}

lib/rocketchat/emoji.js

@@ -0,0 +1,75 @@
+const Factory = require('../factory')
+const Utils = require('./utils')
+
+//
+// Initialize the child logger for
+// the module
+//
+const log = require('../log').child({
+  module: 'custom emoji'
+})
+
+module.exports = async function (context) {
+  const collection = context.rocketchat.emojiCollection()
+  const fileCollection = context.rocketchat.collection('custom_emoji')
+
+  const cursor = collection.find()
+  while (await cursor.hasNext()) {
+    const result = await cursor.next()
+
+    // Prepare destination file
+    const filename = `${result.name}.${result.extension}`
+    const dest = `${context.config.target.filesPath}/custom_emoji/${filename}`
+    // Download emoji file
+    if (result.store.startsWith('FileSystem:')) {
+      const src = context.config.source.customEmojiPath
+      const srcFilename = Utils.srcPath(src, filename)
+      if (!srcFilename) {
+        return new Error(`source file "${filename}" not found`)
+      }
+      // Copy to output dir
+      Utils.copyFile(srcFilename, dest)
+    } else if (result.store.startsWith('GridFS:')) {
+      await Utils.downloadGridFS(context, fileCollection, filename, dest)
+    } else {
+      throw new Error(`file system ${file.store} is not supported. Migrate to FileSystem first, see readme.`)
+    }
+
+    // Export custom emoji
+    let emoji = {
+      name: result.name,
+      image: dest,
+    }
+    log.info(`... writing ${emoji.name}`)
+    context.output.write(
+      Factory.emoji(emoji)
+    )
+  }
+  return context
+}
+
+const exportFile = async function (context, collection, fileId) {
+  const type = collection.collectionName
+  const file = await collection.findOne({ _id: fileId })
+  if (!file) {
+    throw new Error(`file ${fileId} from collection ${type} is missing`)
+  }
+  const dest = context.config.target.filesPath
+  const destFilename = utils.destPath(dest, collection.collectionName, file)
+
+  if (file.store.startsWith('FileSystem:')) {
+    const src = context.config.source.uploadsPath
+    const srcFilename = utils.srcPath(src, file._id)
+    if (!srcFilename) {
+      return new Error(`source file "${file._id}" not found`)
+    }
+    // Copy to output dir
+    utils.copyFile(srcFilename, destFilename)
+  } else if (file.store.startsWith('GridFS:')) {
+    await utils.downloadGridFS(context, collection, file._id, destFilename)
+  } else {
+    throw new Error(`file system ${file.store} is not supported. Migrate to FileSystem first, see readme.`)
+  }
+
+  return destFilename
+}

lib/rocketchat/end.js

@@ -0,0 +1,13 @@
+//
+// Initialize the child logger for
+// the module
+//
+const log = require('../log').child({
+  module: 'end'
+})
+
+module.exports = function(context) {
+  log.info('end')
+  context.rocketchat.close()
+  context.output.end()
+}

lib/rocketchat/index.js

@@ -0,0 +1,21 @@
+const start = require('./start')
+const version = require('./version')
+const emoji = require('./emoji')
+const team = require('./team')
+const channels = require('./channels')
+const users = require('./users')
+const posts = require('./posts')
+const directChannels = require('./directChannels')
+const end = require('./end')
+
+module.exports = {
+  start,
+  version,
+  emoji,
+  team,
+  channels,
+  users,
+  posts,
+  directChannels,
+  end
+}

lib/rocketchat/posts.js

@@ -0,0 +1,162 @@
+const Factory = require('../factory')
+const Utils = require('./utils')
+const _ = require('lodash')
+
+//
+// Initialize the child logger for
+// the module
+//
+const log = require('../log').child({
+  module: 'posts'
+})
+
+module.exports = async function (context) {
+  const collection = context.rocketchat.messagesCollection()
+  // Keep track of the number of posts written for logging
+  let written = 0
+
+  let memReplyIds = {}
+  const total = await collection.count()
+  const cursor = collection.find()
+  while (await cursor.hasNext()) {
+    const result = await cursor.next()
+    let posts = await collectPostData(context, result, memReplyIds, false)
+    posts.forEach(function(post) {
+      context.output.write(post.isDirect ? Factory.directPost(post) : Factory.post(post))
+    })
+    // Log progress periodically
+    written += posts.length
+    if (written % 1000 == 0) {
+      log.info(`... wrote ${written} posts`)
+    }
+  }
+  log.info(`... finished exporting ${written} posts, ignored ${total - written}`)
+  return context
+}
+
+async function collectPostData(context, result, memReplyIds, isReply) {
+  if (memReplyIds.hasOwnProperty(result._id)) {
+    return []
+  }
+
+  try {
+    // Try to get discussion channel id
+    let channelId = result.rid
+    const mergeDiscussions = _.get(context, 'config.define.channels.mergeDiscussionIntoParent', false)
+    const parentId = _.get(context, `values.discussions.${channelId}`, false)
+    if (mergeDiscussions && parentId) {
+      channelId = parentId
+    }
+    // Check if direct message
+    let isDirect = !context.values.channels[channelId] && !!context.values.directChannels[channelId]
+    let post = {}
+    let channelInfo = {}
+
+    if (isDirect) {
+      let { members: channel_members } = context.values.directChannels[channelId]
+      // Ensure we have at least two channel members before we can write the message
+      if (Utils.membersAreValid(channel_members)) {
+        channelInfo = { channel_members }
+      } else {
+        log.error(`... ignoring message id:${result._id} on error: directChannel ${channelId} not found.`)
+        return []
+      }
+    } else {
+      channelInfo = {
+        team: context.values.team.name,
+        channel: Utils.channelName(
+          context.values.channels, channelId
+        ),
+      }
+    }
+    if (!isReply) {
+      Object.assign(post, channelInfo)
+    }
+
+    const reactions = Object.keys(result.reactions || {}).reduce((prev, code) => {
+      return result.reactions[code].usernames.map((u) => {
+        return {
+          user: u,
+          emoji_name: _.trim(code, ':'),
+          create_at: Utils.millis(result.ts),
+        }
+      }).concat(prev)
+    }, [])
+    const flagged_by = (result.starred || []).map(({ _id: uid }) => {
+      try {
+        return Utils.username(context.values.users, uid)
+      } catch (err) {
+        return undefined
+      }
+    }).filter(v => v)
+
+    Object.assign(post, {
+      user: Utils.username(
+        context.values.users, result.u._id
+      ),
+      create_at: Utils.millis(result.ts),
+      reactions,
+      flagged_by,
+      isDirect,
+    })
+
+    // Collect data from attachments
+    let attachments = Utils.processAttachments(context, result)
+    let file
+    let body = result.msg
+
+    await Promise.all(attachments.map(async (a) => {
+      if (a.type === 'file') {
+        file = await a.data
+      } else if (a.type === 'quote') {
+        body = a.data
+      }
+    }))
+    if (file && file instanceof Error) {
+      throw file
+    }
+
+    if (file && file.description) {
+      body = `File description: \n${file.description} \n\n ${body}`
+    }
+    const chunks = body ? Utils.body(body) : [body]
+    let posts = chunks.map((chunk) => {
+      return Object.assign({}, post, {
+        message: chunk
+      })
+    })
+
+    if (!posts[0]) {
+      throw new Error(`Post is empty`)
+    }
+
+    if (file) {
+      posts[0].attachments = [{
+        path: file.path
+      }]
+    }
+
+    const replies = []
+    if (!isReply) {
+      const collection = context.rocketchat.messagesCollection()
+      const cursor = collection.find({tmid: result._id })
+      while (await cursor.hasNext()) {
+        const reply = await cursor.next()
+        const replyData = await collectPostData(context, reply, memReplyIds, true)
+        replyData.forEach(r => replies.push(r))
+        memReplyIds[reply._id] = true
+      }
+    }
+
+    Object.assign(posts[0], {
+      reactions,
+      flagged_by,
+      replies,
+    })
+
+    return posts
+  } catch (err) {
+    log.error(`... ignoring message id:${result._id} on error: ${err.message}.`)
+    return []
+  }
+}

lib/rocketchat/start.js

@@ -0,0 +1,32 @@
+const datafile = require('../datafile')
+const fs = require('fs');
+
+//
+// Initialize the child logger for
+// the module
+//
+const log = require('../log').child({
+  module: 'start'
+})
+
+module.exports = async function(context) {
+  log.info('preparing files paths')
+  const dest = context.config.target.filesPath
+  if (!dest || !fs.existsSync(dest)) {
+    fs.mkdirSync(dest)
+  }
+  if (!fs.lstatSync(dest).isDirectory() || fs.accessSync(dest, fs.constants.W_OK)) {
+    throw new Error(`Directory "${dest} is not writable"`)
+  }
+
+  log.info('connecting to rocketchat')
+  await context.rocketchat.connect(context.config)
+  log.info(`creating file '${context.config.target.filename}'`)
+  // Create the datafile and add it to the context
+  context.output = datafile(
+    context.config.target.filename,
+    process.exit
+  )
+
+  return context
+}

lib/rocketchat/team.js

@@ -0,0 +1,29 @@
+const Factory = require('../factory')
+
+//
+// Initialize the child logger for
+// the module
+//
+const log = require('../log').child({
+  module: 'team'
+})
+
+module.exports = function(context) {
+  log.info(`writing team '${context.config.define.team.name}'`)
+  //
+  // Store the team object for use
+  // by other modules
+  //
+  context.values.team = context.config.define.team
+  //
+  // Write the team object to the
+  // output
+  //
+  context.output.write(Factory.team(
+    context.config.define.team
+  ))
+  //
+  // Return a resolved promise
+  //
+  return Promise.resolve(context)
+}

lib/rocketchat/users.js

@@ -0,0 +1,105 @@
+const Factory = require('../factory')
+const Utils = require('./utils')
+const { Gitlab } = require('@gitbeaker/node')
+const _ = require('lodash')
+
+//
+// Initialize the child logger for
+// the module
+//
+const log = require('../log').child({
+  module: 'users'
+})
+
+module.exports = async function(context) {
+  const collection = context.rocketchat.usersCollection()
+
+  let users = {}
+  const cursor = collection.find({ type: 'user' })
+  while (await cursor.hasNext()) {
+    const result = await cursor.next()
+    const id = result._id
+    const [first_name, last_name] = result.name.split(' ')
+    try {
+      const ldapAuthService = _.get(context, 'config.define.user.ldap_auth_service', '')
+      const roleMap = _.get(context, 'config.define.user.globalRoleMap', {})
+      const auth_service = result.ldap ? ldapAuthService : ''
+      if (result.roles.indexOf('admin') !== -1 && result.roles.indexOf('user') === -1) {
+        result.roles.push('user')
+      }
+      const highlights = _.get(result, 'settings.preferences.highlights', [])
+      let user = users[id] = {
+        username: result.username,
+        first_name,
+        last_name,
+        auth_service,
+        auth_data: await getAuthData(context, result, auth_service),
+        email: result.emails[0].address,
+        roles: result.roles.sort().reduce((roles, role) => {
+          if (roleMap.hasOwnProperty(role)) {
+            roles.push(roleMap[role])
+          }
+          return roles
+        }, []).join(' '),
+        teams: [{
+          name: context.values.team.name,
+          channels: result.__rooms.map((id) => {
+            const name = _.get(context, `values.channels.${id}.name`, false)
+            return name ? { name } : undefined
+          }).filter(v => v)
+        }],
+        notify_props: {
+          mention_keys: highlights.map(s => s.replace('@', '')).join(','),
+        },
+      }
+      if (auth_service) {
+        user.auth_service = auth_service
+        user.auth_data = await getAuthData(context, result, auth_service)
+      }
+      if (!user.roles.length) {
+        delete user.roles
+      }
+      const avatar = await getAvatar(context, result)
+      if (avatar) {
+        user.profile_image = avatar
+      }
+
+      context.output.write(
+        Factory.user(user)
+      )
+      log.info(`... writing ${user.username}`)
+    }
+    catch(err) {
+      log.error(`... ignoring ${result.username} on error: ${err.message}.`)
+      delete users[id]
+    }
+  }
+  context.values.users = users
+  return context
+}
+
+async function getAuthData(context, user, auth_service) {
+  switch (auth_service) {
+    case 'gitlab':
+      const {host, token} = _.get(context, 'config.define.user.gitlab')
+      const api = new Gitlab({ host, token });
+      const gitlabUser = await api.Users.username(user.username);
+      if (!gitlabUser || !gitlabUser.length) {
+        throw new Error(`user ${user.username} is missing in Gitlab`)
+      }
+      return `${gitlabUser[0].id}`
+    case 'ldap':
+      return user.username.toUpperCase()
+    default:
+      return ''
+  }
+}
+
+async function getAvatar(context, user) {
+  const collection = context.rocketchat.avatarsCollection()
+  const avatar = await collection.findOne({userId: user._id}, {sort: {_updatedAt: -1}})
+  if (!avatar) {
+    return ""
+  }
+  return Utils.exportFile(context, collection, avatar._id)
+}

lib/rocketchat/utils.js

@@ -0,0 +1,191 @@
+const _ = require('lodash')
+const fs = require('fs')
+const path = require('path')
+
+//
+// Declare utils object
+//
+const utils = {}
+
+//
+//
+//
+utils.chunk = function(body) {
+  //
+  // Use regex to create an array
+  // of strings of max length
+  //
+  return body.match(/[\s\S]{1,4000}/g)
+}
+
+//
+// Lookup values
+//
+utils.lookup = function (type, map, key) {
+  var found = map[key]
+
+  if(!found) {
+    throw new Error(`${type} ${key} not found`)
+  }
+
+  return found
+}
+
+//
+// Obtain the username from a jid
+//
+utils.username = function (users, id) {
+  return utils.lookup('user', users, id).username
+}
+
+//
+// Obtain the channel name from a jid
+//
+utils.channelName = function (channels, id) {
+  return utils.lookup('channel', channels, id).name
+}
+
+//
+// Find the message body
+//
+utils.body = function (body) {
+  return utils.chunk(body)
+}
+
+utils.processAttachments = function (context, message) {
+  if (!message.attachments) {
+    return []
+  }
+  return message.attachments.map((attachment) => {
+    if (attachment.type && attachment.type === 'file') {
+      return {
+        type: 'file',
+        data: utils.processFileAttachment(context, message, attachment),
+      }
+    }
+    if (attachment.text) {
+      return {
+        type: 'quote',
+        data: utils.processQuoteAttachment(message, attachment)
+      }
+    }
+    return {
+      type: 'unknown'
+    }
+  })
+}
+
+utils.processQuoteAttachment = function (message, attachment) {
+  if (!attachment.text) {
+    return ''
+  }
+  const body = (message.msg || '').replace(/\[ \]\(.*msg=.*?\) /, '')
+  // Convert quote to markdown
+  return `@${attachment.author_name}:\n ${attachment.text.replace(/^/, '> ')} \n\n${body}`
+}
+
+utils.processFileAttachment = async function (context, message, attachment) {
+  if (!message.file) {
+    return new Error(`message ${message._id} file is missing`)
+  }
+  const collection = context.rocketchat.uploadsCollection()
+  return {
+    description: attachment.description,
+    path: await utils.exportFile(context, collection, message.file._id),
+  }
+}
+
+utils.srcPath = function (srcDir, filename) {
+  const files = fs.readdirSync(srcDir).filter((fn) => fn.startsWith(filename));
+  if (files.length !== 1) {
+    return false
+  }
+  return `${srcDir}/${path.basename(files[0])}`
+}
+
+utils.destPath = function (destDir, type, file) {
+  let dest = `${destDir}/${type}/${file._id}/${file.name}`;
+  const ext = path.extname(dest)
+  if (!ext || !/^\.[A-Za-z][A-Za-z0-9]*$/.test(ext)) {
+    if (file.identify && file.identify.format) {
+      dest += '.' + file.identify.format
+    } else {
+      dest += '.' + file.type.split('/')[1]
+    }
+  }
+  return dest
+}
+
+utils.exportFile = async function (context, collection, fileId) {
+  const type = collection.collectionName
+  const file = await collection.findOne({ _id: fileId })
+  if (!file) {
+    throw new Error(`file ${fileId} from collection ${type} is missing`)
+  }
+  const dest = context.config.target.filesPath
+  const destFilename = utils.destPath(dest, collection.collectionName, file)
+
+  if (file.store.startsWith('FileSystem:')) {
+    const src = context.config.source.uploadsPath
+    const srcFilename = utils.srcPath(src, file._id)
+    if (!srcFilename) {
+      return new Error(`source file "${file._id}" not found`)
+    }
+    // Copy to output dir
+    utils.copyFile(srcFilename, destFilename)
+  } else if (file.store.startsWith('GridFS:')) {
+    await utils.downloadGridFS(context, collection, file._id, destFilename)
+  } else {
+    throw new Error(`file system ${file.store} is not supported. Migrate to FileSystem first, see readme.`)
+  }
+
+  return destFilename
+}
+
+utils.copyFile = function (src, dest) {
+  if (!fs.existsSync(path.dirname(dest))) {
+    fs.mkdirSync(path.dirname(dest), { recursive: true })
+  }
+  if (!fs.existsSync(dest)) {
+    fs.copyFileSync(src, dest)
+  }
+}
+
+utils.downloadGridFS = async function (context, collection, id, dest) {
+  if (!fs.existsSync(path.dirname(dest))) {
+    fs.mkdirSync(path.dirname(dest), { recursive: true })
+  }
+  if (fs.existsSync(dest)) {
+    fs.unlinkSync(dest)
+  }
+
+  const bucket = context.rocketchat.gridFsBucket(collection.collectionName)
+  const destStream = fs.createWriteStream(dest);
+  bucket.openDownloadStream(id).pipe(destStream);
+  return new Promise((resolve, reject) => {
+    destStream.on('finish', resolve);
+  })
+}
+
+utils.members = function(users, usernames) {
+  return _.uniq(_.sortBy(usernames.map((username) => utils.username(users, username))))
+}
+
+//
+// Checks if the members list is valid
+//
+utils.membersAreValid = function(members) {
+  return _.isArray(members) && members.length > 0
+}
+
+//
+// Convert ISO to millis
+//
+utils.millis = function(date) {
+  return new Date(date).getTime()
+}
+
+//
+// Export the functions
+//
+module.exports = utils

lib/rocketchat/version.js

@@ -0,0 +1,12 @@
+const Factory = require('../factory')
+
+module.exports = function(context) {
+  //
+  // Write the version object
+  //
+  context.output.write(Factory.version())
+  //
+  // Return a resolved promise
+  //
+  return Promise.resolve(context)
+}