Changed search system to keyset-based

package.json

@@ -32,6 +32,7 @@
   "dependencies": {
     "@babel/plugin-proposal-optional-chaining": "^7.12.7",
     "babel": "^6.23.0",
+    "base64url": "^3.0.1",
     "bcrypt": "^5.0.1",
     "cloudinary": "^1.24.0",
     "dotenv-flow": "^3.0.0",

src/api/middleware/company.js

@@ -43,7 +43,7 @@ export const verifyMaxConcurrentOffersOnEdit = async (req, res, next) => {
 
     try {
 
-        const offer = await (new OfferService()).getOfferById(req.params.offerId, req.user);
+        const offer = await (new OfferService()).getOfferById(req.params.offerId, req.targetOwner, req.hasAdminPrivileges);
 
         if (!offer)
             throw new APIError(HTTPStatus.NOT_FOUND, ErrorTypes.VALIDATION_ERROR, ValidationReasons.OFFER_NOT_FOUND(req.params.offerId));

src/api/middleware/validators/offer.js

@@ -312,21 +312,23 @@ const publishEndDateEditableLimit = async (publishEndDateCandidate, { req }) =>
     return true;
 };
 
+const existingOfferId = async (offerId, { req }) => {
+    try {
+        const offer = await (new OfferService()).getOfferById(offerId, req.targetOwner, req.hasAdminPrivileges);
+        if (!offer) throw new Error(ValidationReasons.OFFER_NOT_FOUND(offerId));
+    } catch (err) {
+        console.error(err);
+        throw err;
+    }
+
+    return true;
+};
+
 export const isExistingOffer = useExpressValidators([
     param("offerId", ValidationReasons.DEFAULT)
         .exists().withMessage(ValidationReasons.REQUIRED).bail()
         .custom(isObjectId).withMessage(ValidationReasons.OBJECT_ID).bail()
-        .custom(async (offerId, { req }) => {
-            try {
-                const offer = await (new OfferService()).getOfferById(offerId, req.targetOwner, req.hasAdminPrivileges);
-                if (!offer) throw new Error(ValidationReasons.OFFER_NOT_FOUND(offerId));
-            } catch (err) {
-                console.error(err);
-                throw err;
-            }
-
-            return true;
-        }),
+        .custom(existingOfferId),
 ]);
 
 export const edit = useExpressValidators([
@@ -440,11 +442,30 @@ export const setDefaultValuesCreate = (req, res, next) => {
     return next();
 };
 
+const validGetQueryToken = async (queryToken, { req }) => {
+    try {
+        const { id, score } = (new OfferService()).decodeQueryToken(queryToken);
+        if (!isObjectId(id)) throw new Error(ValidationReasons.OBJECT_ID);
+        await existingOfferId(id, { req });
+
+        if (req.query.value) {
+            if (isNaN(score)) throw new Error(ValidationReasons.NUMBER);
+            if (score < 0) throw new Error(ValidationReasons.MIN(0));
+        }
+
+    } catch (err) {
+        console.error(err);
+        throw new Error(ValidationReasons.INVALID_QUERY_TOKEN);
+    }
+
+    return true;
+};
+
 export const get = useExpressValidators([
-    query("offset", ValidationReasons.DEFAULT)
+    query("queryToken", ValidationReasons.DEFAULT)
         .optional()
-        .isInt({ min: 0 }).withMessage(ValidationReasons.INT)
-        .toInt(),
+        .isString().withMessage(ValidationReasons.STRING).bail()
+        .custom(validGetQueryToken),
 
     query("limit")
         .optional()

src/api/middleware/validators/validationReasons.js

@@ -19,6 +19,7 @@ const ValidationReasons = Object.freeze({
     DATE: "must-be-ISO8601-date",
     INT: "must-be-int",
     BOOLEAN: "must-be-boolean",
+    NUMBER: "must-be-number",
     IN_ARRAY: (vals, field) => `${field ? `${field}:` : ""}must-be-in:[${vals}]`,
     ARRAY_SIZE: (min, max) => `size-must-be-between:[${min},${max}]`,
     OBJECT_ID: "must-be-a-valid-id",
@@ -37,6 +38,7 @@ const ValidationReasons = Object.freeze({
     OFFER_EXPIRED: (id) => `offer-expired:${id}`,
     NOT_OFFER_OWNER: (id) => `not-offer-owner:${id}`,
     OFFER_EDIT_PERIOD_OVER: (value) => `offer-edit-period-over:${value}-hours`,
+    INVALID_QUERY_TOKEN: "invalid-query-token",
     JOB_MIN_DURATION_NOT_SPECIFIED: "job-max-duration-requires-job-min-duration",
     REGISTRATION_FINISHED: "registration-already-finished",
     REGISTRATION_NOT_FINISHED: "registration-not-finished-yet",

src/api/routes/offer.js

@@ -20,20 +20,21 @@ export default (app) => {
     router.use(offerMiddleware.setTargetOwner);
 
     /**
-     * Gets all currently active offers (without filtering, for now)
-     * supports offset and limit as query params
+     * Gets active offers based on passed filters and full-text search.
+     * Returns the offers found and a queryToken used for pagination.
+     * Also takes queryToken and limit as query params.
      */
     router.get("/", validators.get, async (req, res, next) => {
         try {
-            const offers = await (new OfferService()).get(
+            const resultsAndQueryToken = await (new OfferService()).get(
                 {
                     ...req.query,
                     showHidden: req?.query?.showHidden && req.hasAdminPrivileges,
                     showAdminReason: req.hasAdminPrivileges
                 }
             );
 
-            return res.json(offers);
+            return res.json(resultsAndQueryToken);
         } catch (err) {
             console.error(err);
             return next(err);

src/models/Offer.js

@@ -146,22 +146,24 @@ function validateDescription(value) {
 /**
  * Currently active Offers (publish date was before Date.now and end date is after Date.now)
  */
+OfferSchema.statics.filterCurrent = () => ({
+    publishDate: {
+        $lte: new Date(Date.now()),
+    },
+    publishEndDate: {
+        $gt: new Date(Date.now()),
+    },
+});
 OfferSchema.query.current = function() {
-    return this.where({
-        publishDate: {
-            $lte: new Date(Date.now()),
-        },
-        publishEndDate: {
-            $gt: new Date(Date.now()),
-        },
-    });
+    return this.where(this.model.filterCurrent());
 };
 
 /**
  * Currently active and non-hidden Offers
  */
+OfferSchema.statics.filterNonHidden = () => ({ isHidden: false });
 OfferSchema.query.withoutHidden = function() {
-    return this.where({ isHidden: false });
+    return this.where(this.model.filterNonHidden());
 };
 
 const Offer = mongoose.model("Offer", OfferSchema);

src/services/offer.js

@@ -1,9 +1,13 @@
+import mongoose from "mongoose";
 import Company  from "../models/Company.js";
 import Offer  from "../models/Offer.js";
 import Account  from "../models/Account.js";
 import EmailService  from "../lib/emailService.js";
 import { OFFER_DISABLED_NOTIFICATION }  from "../email-templates/companyOfferDisabled.js";
 import OfferConstants  from "../models/constants/Offer.js";
+import base64url from "base64url";
+
+const { ObjectId } = mongoose.Types;
 
 class OfferService {
     // TODO: Use typedi or similar
@@ -192,29 +196,91 @@ class OfferService {
 
     /**
      * Fetches offers according to specified options
+     * Learn more about keyset search here: https://github.com/NIAEFEUP/nijobs-be/issues/129
+     *
      * @param {*} options
      * value: Text to use in full-text-search
-     * offset: Point to start looking (and limiting)
+     * queryToken: Id of the last fetched offer
      * limit: How many offers to show
      * jobType: Array of jobTypes allowed
      */
-    get({ value = "", offset = 0, limit = OfferService.MAX_OFFERS_PER_QUERY, showHidden = false, showAdminReason = false, ...filters }) {
+    async get({ value = "", queryToken = null, limit = OfferService.MAX_OFFERS_PER_QUERY,
+        showHidden = false, showAdminReason = false, ...filters }) {
 
-        const offers = (value ? Offer.find(
-            { "$and": [this._buildFilterQuery(filters), { "$text": { "$search": value } }] }, { score: { "$meta": "textScore" } }
-        ) : Offer.find(this._buildFilterQuery(filters))).current();
+        const {
+            id: lastOfferId,
+            score: lastOfferScore
+        } = queryToken ? this.decodeQueryToken(queryToken) : {};
 
-        if (!showHidden) offers.withoutHidden();
+        const offers = lastOfferId && value ?
+            this._buildSearchAggregation(lastOfferId, lastOfferScore, value, showAdminReason, showHidden, filters)
+            :
+            this._buildSearchQuery(lastOfferId, value, showAdminReason, showHidden, filters);
 
-        const offersQuery = offers
-            .sort(value ? { score: { "$meta": "textScore" } } : undefined)
-            .skip(offset)
+        const results = await offers
+            .sort(value ? { score: { "$meta": "textScore" }, _id: 1 } : { _id: 1 })
             .limit(limit)
         ;
 
-        return showAdminReason ? offersQuery : offersQuery.select("-adminReason");
+        return results.length > 0 ?
+            { results, queryToken: this.encodeQueryToken(results[results.length - 1]) }
+            :
+            { results };
+    }
+
+    /**
+     * Builds a search query with a regular find().
+     * Cannot be used when loading more offers and searching with full-text search at the same time.
+     * Otherwise, use _buildSearchAggregation().
+     */
+    _buildSearchQuery(lastOfferId, value, showAdminReason, showHidden, filters) {
+        let offers;
+        if (lastOfferId) {
+
+            offers = Offer.find({ "$and": [
+                this._buildFilterQuery(filters),
+                { _id: { "$gt": ObjectId(lastOfferId) } }
+            ] });
+
+        } else {
 
+            offers = (value ? Offer.find({ "$and": [
+                this._buildFilterQuery(filters),
+                { "$text": { "$search": value } }
+            ] }, { score: { "$meta": "textScore" } }
+
+            ) : Offer.find(this._buildFilterQuery(filters)));
+        }
+
+        offers.current();
+        if (!showHidden) offers.withoutHidden();
+        if (!showAdminReason) offers.select("-adminReason");
+
+        return offers;
     }
+
+    /**
+     * Builds a search aggregation with aggregate().
+     * Only use this when loading more offers and searching with full-text search at the same time.
+     * Otherwise, use _buildSearchQuery().
+     */
+    _buildSearchAggregation(lastOfferId, lastOfferScore, value, showAdminReason, showHidden, filters) {
+        return Offer.aggregate([
+            { $match: { $text: { $search: value } } },
+            { $match: filters },
+            { $addFields: {
+                score: { $meta: "textScore" },
+                adminReason: { $cond: [showAdminReason, "$adminReason", "$$REMOVE"] }
+            } },
+            { $match: { "$or": [
+                { score: { "$lt": lastOfferScore } },
+                { score: lastOfferScore, _id: { "$gt": ObjectId(lastOfferId) } }
+            ] } },
+            { $match: Offer.filterCurrent() },
+            { $match: showHidden ? {} : Offer.filterNonHidden() }
+        ]);
+    }
+
     _buildFilterQuery(filters) {
         if (!filters || !Object.keys(filters).length) return {};
 
@@ -262,6 +328,30 @@ class OfferService {
         return constraints.length ? { "$and": constraints } : {};
     }
 
+    /**
+     * Encodes a query token, by taking the offer's id and FTS score (if present) and decoding in safe url base64
+     * @param {*} offer
+     */
+    encodeQueryToken(offer) {
+        return base64url.encode(JSON.stringify({
+            id: offer._id,
+            score: offer.score || offer._doc?.score
+        }));
+    }
+
+    /**
+     * Decodes a query token, extracting the lastOffer's ID and FTS score
+     * @param {*} queryToken
+     */
+    decodeQueryToken(queryToken) {
+        const tokenInfo = JSON.parse(base64url.decode(queryToken));
+
+        return {
+            ...tokenInfo,
+            score: Number(tokenInfo.score)
+        };
+    }
+
     /**
      * Checks whether a given offer is visible to a specific userCompanyId.
      * Unpublished/Unactive offers may still be visible