Merge pull request #1091 from CVEProject/jd-741

Resolves #741 Fixes GET `/cve-id` endpoint's out of memory bug

src/constants/index.js

@@ -79,6 +79,7 @@ function getConstants () {
     PAGINATOR_PAGE: 1,
     PAGINATOR_OPTIONS: {
       limit: 500,
+      lean: true,
       useFacet: false,
       customLabels: {
         totalDocs: 'itemCount',

src/controller/cve-id.controller/cve-id.controller.js

@@ -33,9 +33,26 @@ async function getFilteredCveId (req, res, next) {
     options.page = req.ctx.query.page ? parseInt(req.ctx.query.page) : CONSTANTS.PAGINATOR_PAGE // if 'page' query parameter is not defined, set 'page' to the default page value
     const cveIdRepo = req.ctx.repositories.getCveIdRepository()
     const orgRepo = req.ctx.repositories.getOrgRepository()
+    const userRepo = req.ctx.repositories.getUserRepository()
     const isSecretariat = await orgRepo.isSecretariat(orgShortName)
     const isBulkDownload = await orgRepo.isBulkDownload(orgShortName)
 
+    // Create map of orgUUID to shortnames and users to simplify aggregation later
+    const orgs = await orgRepo.getAllOrgs()
+    const users = await userRepo.getAllUsers()
+
+    const orgMap = {}
+    orgs.forEach(org => {
+      orgMap[org.UUID] = { shortname: org.short_name, users: {} }
+    })
+
+    users.forEach(user => {
+      if (!orgMap[user.org_UUID]) {
+        orgMap[user.org_UUID] = { shortname: `MISSING ORG ${user.org_UUID}`, users: {} }
+      }
+      orgMap[user.org_UUID].users[user.UUID] = user.username
+    })
+
     Object.keys(req.ctx.query).forEach(k => {
       const key = k.toLowerCase()
 
@@ -99,9 +116,17 @@ async function getFilteredCveId (req, res, next) {
       }
     }
 
-    const agt = setAggregateObj(query)
+    const agt = setMinAggregateObj(query)
     const pg = await cveIdRepo.aggregatePaginate(agt, options)
-    const payload = { cve_ids: pg.itemsList }
+    const payload = {
+      cve_ids: pg.itemsList.map((i) => {
+        const cnaid = i.requested_by.cna
+        i.requested_by.cna = orgMap[cnaid].shortname
+        i.requested_by.user = orgMap[cnaid].users[i.requested_by.user]
+        i.owning_cna = orgMap[i.owning_cna].shortname
+        return i
+      })
+    }
 
     if (pg.itemCount >= CONSTANTS.PAGINATOR_OPTIONS.limit) {
       payload.totalCount = pg.itemCount
@@ -876,6 +901,22 @@ function setAggregateObj (query) {
   ]
 }
 
+// Smaller aggregation used in getFilteredCveId function
+function setMinAggregateObj (query) {
+  return [
+    {
+      $match: query
+    },
+
+    {
+      $project: {
+        _id: false,
+        __v: false
+      }
+    }
+  ]
+}
+
 module.exports = {
   CVEID_GET_FILTER: getFilteredCveId,
   CVEID_RESERVE: reserveCveId,

src/repositories/orgRepository.js

@@ -34,6 +34,10 @@ class OrgRepository extends BaseRepository {
   async isBulkDownload (shortName) {
     return utils.isBulkDownload(shortName)
   }
+
+  async getAllOrgs () {
+    return this.collection.find()
+  }
 }
 
 module.exports = OrgRepository

src/repositories/userRepository.js

@@ -30,6 +30,10 @@ class UserRepository extends BaseRepository {
   async updateByUserNameAndOrgUUID (username, orgUUID, user, options = {}) {
     return this.collection.findOneAndUpdate().byUserNameAndOrgUUID(username, orgUUID).updateOne(user).setOptions(options)
   }
+
+  async getAllUsers () {
+    return this.collection.find()
+  }
 }
 
 module.exports = UserRepository

test/integration-tests/cve-id/getCveIdTest.js

@@ -0,0 +1,103 @@
+/* eslint-disable no-unused-expressions */
+
+const chai = require('chai')
+chai.use(require('chai-http'))
+const _ = require('lodash')
+
+const expect = chai.expect
+
+const constants = require('../constants.js')
+const app = require('../../../src/index.js')
+
+describe('Testing Get CVE-ID endpoint', () => {
+  const RESESRVED_COUNT = 116
+  const YEAR_COUNT = 10
+  const PUB_YEAR_COUNT = 4
+  const TIME_WINDOW_COUNT = 40
+
+  context('Positive Tests', () => {
+    it('Get CVE-ID should return everything when no parameters are specifed', async () => {
+      await chai.request(app)
+        .get('/api/cve-id')
+        .set(constants.headers)
+        .then((res, err) => {
+          expect(err).to.be.undefined
+          expect(res).to.have.status(200)
+        })
+    })
+    it('Get CVE-ID should return an empty array when time modified is set to a very far future date', async () => {
+      await chai.request(app)
+        .get('/api/cve-id?time_modified.gt=2100-01-01T00:00:00')
+        .set(constants.headers)
+        .then((res, err) => {
+          expect(err).to.be.undefined
+          expect(res).to.have.status(200)
+          expect(res.body.cve_ids).to.have.length(0)
+        })
+    })
+    // Need a better way to test each individual cve-id's time_modified
+    it('Get all CVE-IDs modified within a given timeframe', async () => {
+      await chai.request(app)
+        .get('/api/cve-id?time_modified.gt=2021-05-11T15:05:20.093Z&time_modified.lt=2021-05-11T15:07:00.093Z')
+        .set(constants.headers)
+        .then((res, err) => {
+          expect(err).to.be.undefined
+          expect(res).to.have.status(200)
+          expect(res.body.cve_ids).to.have.length(TIME_WINDOW_COUNT)
+        })
+    })
+    it('Get all CVE-IDs in the RESERVED state', async () => {
+      await chai.request(app)
+        .get('/api/cve-id?state=RESERVED')
+        .set(constants.headers)
+        .then((res, err) => {
+          expect(err).to.be.undefined
+          expect(res).to.have.status(200)
+          expect(_.every(res.body.cve_ids, { state: 'RESERVED' })).to.be.true
+          expect(res.body.cve_ids).to.have.length(RESESRVED_COUNT)
+        })
+    })
+    it('Get all CVE-IDs in the PUBLISHED state', async () => {
+      await chai.request(app)
+        .get('/api/cve-id?state=PUBLISHED')
+        .set(constants.headers)
+        .then((res, err) => {
+          expect(err).to.be.undefined
+          expect(res).to.have.status(200)
+          expect(_.every(res.body.cve_ids, { state: 'PUBLISHED' })).to.be.true
+        })
+    })
+    it('Get all CVE-IDs in the REJECTED state', async () => {
+      await chai.request(app)
+        .get('/api/cve-id?state=REJECTED')
+        .set(constants.headers)
+        .then((res, err) => {
+          expect(err).to.be.undefined
+          expect(res).to.have.status(200)
+          expect(_.every(res.body.cve_ids, { state: 'REJECTED' })).to.be.true
+        })
+    })
+    it('Get all CVE-IDs with cve_id_year 1999', async () => {
+      await chai.request(app)
+        .get('/api/cve-id?cve_id_year=1999')
+        .set(constants.headers)
+        .then((res, err) => {
+          expect(err).to.be.undefined
+          expect(res).to.have.status(200)
+          expect(_.every(res.body.cve_ids, { cve_year: '1999' })).to.be.true
+          expect(res.body.cve_ids).to.have.length(YEAR_COUNT)
+        })
+    })
+    it('Get all CVE-IDs with cve_id_year 1999 with state PUBLISHED', async () => {
+      await chai.request(app)
+        .get('/api/cve-id?cve_id_year=1999&state=PUBLISHED')
+        .set(constants.headers)
+        .then((res, err) => {
+          expect(err).to.be.undefined
+          expect(res).to.have.status(200)
+          expect(_.every(res.body.cve_ids, { cve_year: '1999', state: 'PUBLISHED' })).to.be.true
+          expect(res.body.cve_ids).to.have.length(PUB_YEAR_COUNT)
+        })
+    })
+  })
+})