feat: implement caching of api responses

should help a bit in not getting rate-limited

.env.example

@@ -8,6 +8,9 @@ NODE_ENV=production
 ## user agent and accept header that quora will see
 # AXIOS_USER_AGENT='axios/0.26.1'
 # AXIOS_ACCEPT='application/json, text/plain, */*'
+## for caching api responses using redis
+# REDIS_URL=localhost:6379 # if left unset, there'll be no caching
+# REDIS_TTL=3600
 
 ### for specific use-cases.
 ## add any value here (e.g.: 1, true, 'por favor') if you're using any service where http is the preferred method(e.g.: tor, i2p). else leave it blank

.gitignore

@@ -2,4 +2,5 @@
 node_modules/*
 .env
 dev-data/*
-public/css/*
+public/css/*
+dump.rdb

README.md

@@ -188,7 +188,7 @@ From [their privacy policy](https://www.quora.com/about/privacy)
 ## To-Do
 
 - [x] add missing routes like topics, profile, and search
-- [ ] use redis
+- [x] use redis
 - [x] serve images and other assets from Quetre
 - [x] implement a better installation method
 - [ ] implement other trivial routes like a specific answer, spaces, etc.
@@ -200,7 +200,7 @@ From [their privacy policy](https://www.quora.com/about/privacy)
 
 ### Manual
 
-1. Install [Node.js](https://nodejs.org/en/) and [Git](https://git-scm.com/). Instructions are on their websites.
+1. Install [Node.js](https://nodejs.org/en/), [Git](https://git-scm.com/), and [Redis](https://redis.io)(Optional). Instructions are on their websites.
 
 2. Clone and set up the repository.
 
@@ -211,6 +211,8 @@ From [their privacy policy](https://www.quora.com/about/privacy)
    # change `pnpm` to `npm run` here as well as in package.json if you use `npm`
    pnpm install
    pnpm start
+   # optional
+   redis-server # useful for caching api responses
    ```
 
 Quetre will start running at http://localhost:3000.

controllers/apiController.js

@@ -8,6 +8,8 @@ import getAnswers from '../fetchers/getAnswers.js';
 import getTopic from '../fetchers/getTopic.js';
 import getProfile from '../fetchers/getProfile.js';
 import getSearch from '../fetchers/getSearch.js';
+import getOrSetCache from '../utils/getOrSetCache.js';
+import { answersKey, profileKey, searchKey, topicKey } from '../utils/cacheKeys.js';
 
 ////////////////////////////////////////////////////////
 //                     EXPORTS
@@ -21,31 +23,49 @@ export const about = (req, res, next) => {
 };
 
 export const answers = catchAsyncErrors(async (req, res, next) => {
-	const { slug } = req.params;
-	const { lang } = req.query;
+  const {
+    urlObj,
+    params: { slug },
+    query: { lang },
+  } = req;
 
-  const data = await getAnswers(slug, lang);
+  const data = await getOrSetCache(answersKey(urlObj), getAnswers, slug, lang);
   res.status(200).json({ status: 'success', data });
 });
 
 export const topic = catchAsyncErrors(async (req, res, next) => {
-	const { slug } = req.params;
-	const { lang } = req.query;
+  const {
+    urlObj,
+    params: { slug },
+    query: { lang },
+  } = req;
 
-  const data = await getTopic(slug, lang);
+  const data = await getOrSetCache(topicKey(urlObj), getTopic, slug, lang);
   res.status(200).json({ status: 'success', data });
 });
 
 export const profile = catchAsyncErrors(async (req, res, next) => {
-  const data = await getProfile(req.params.name);
+  const {
+    urlObj,
+    params: { name },
+    query: { lang },
+  } = req;
+
+  const data = await getOrSetCache(profileKey(urlObj), getProfile, name, lang);
   res.status(200).json({ status: 'success', data });
 });
 
 export const search = catchAsyncErrors(async (req, res, next) => {
-  const searchText = req.urlObj.searchParams.get('q')?.trim(); // no search to perform if there isn't any query
+  const {
+    urlObj,
+    query: { lang },
+  } = req;
 
+  const searchText = urlObj.searchParams.get('q')?.trim(); // no search to perform if there isn't any query
   let searchData = null;
-  if (searchText) searchData = await getSearch(req.urlObj.search);
+
+  if (searchText)
+    searchData = await getOrSetCache(searchKey(urlObj), getSearch, urlObj.search, lang);
 
   res.status(200).json({ status: 'success', data: searchData });
 });
@@ -72,5 +92,6 @@ export const image = catchAsyncErrors(async (req, res, next) => {
   const imageRes = await axiosInstance.get(path, { responseType: 'stream' });
 
   res.set('Content-Type', imageRes.headers['content-type']);
+  res.set('Cache-Control', 'public, max-age=315360000');
   return imageRes.data.pipe(res);
 });

controllers/viewController.js

@@ -8,6 +8,8 @@ import getTopic from '../fetchers/getTopic.js';
 import { acceptedLanguages, nonSlugRoutes } from '../utils/constants.js';
 import getProfile from '../fetchers/getProfile.js';
 import getSearch from '../fetchers/getSearch.js';
+import getOrSetCache from '../utils/getOrSetCache.js';
+import { answersKey, profileKey, searchKey, topicKey } from '../utils/cacheKeys.js';
 
 ////////////////////////////////////////////////////////
 //                     EXPORTS
@@ -36,16 +38,17 @@ export const privacy = (req, res, next) => {
 };
 
 export const answers = catchAsyncErrors(async (req, res, next) => {
-  const { slug } = req.params;
-  const { lang } = req.query;
+  const {
+    urlObj,
+    params: { slug },
+    query: { lang },
+  } = req;
 
   // added this so that a request by browser to get favicon doesn't end up being interpreted as a slug
   if (nonSlugRoutes.includes(slug)) return next();
 
-  const answersData = await getAnswers(slug, lang);
-  const title = answersData.question.text[0].spans
-    .map(span => span.text)
-    .join('');
+  const answersData = await getOrSetCache(answersKey(urlObj), getAnswers, slug, lang);
+  const title = answersData.question.text[0].spans.map(span => span.text).join('');
 
   return res.status(200).render('answers', {
     data: answersData,
@@ -59,50 +62,62 @@ export const answers = catchAsyncErrors(async (req, res, next) => {
 });
 
 export const topic = catchAsyncErrors(async (req, res, next) => {
-  const { slug } = req.params;
-  const { lang } = req.query;
+  const {
+    urlObj,
+    params: { slug },
+    query: { lang },
+  } = req;
 
-  const topicData = await getTopic(slug, lang);
+  const topicData = await getOrSetCache(topicKey(urlObj), getTopic, slug, lang);
 
   res.status(200).render('topic', {
     data: topicData,
     meta: {
       title: topicData.name,
-      url: req.urlObj,
-      imageUrl: `${req.urlObj.origin}/icon.svg`,
+      url: urlObj,
+      imageUrl: `${urlObj.origin}/icon.svg`,
       description: `Information about ${topicData.name} topic.`,
     },
   });
 });
 
 export const profile = catchAsyncErrors(async (req, res, next) => {
-  const { name } = req.params;
-  const { lang } = req.query;
-  const profileData = await getProfile(name, lang);
+  const {
+    urlObj,
+    params: { name },
+    query: { lang },
+  } = req;
+
+  const profileData = await getOrSetCache(profileKey(urlObj), getProfile, name, lang);
 
   res.status(200).render('profile', {
     data: profileData,
     meta: {
       title: profileData.basic.name,
-      url: req.urlObj,
-      imageUrl: `${req.urlObj.origin}/icon.svg`,
+      url: urlObj,
+      imageUrl: `${urlObj.origin}/icon.svg`,
       description: `${profileData.basic.name}'s profile.`,
     },
   });
 });
 
 export const search = catchAsyncErrors(async (req, res, next) => {
-  const searchText = req.urlObj.searchParams.get('q')?.trim();
-  const { lang } = req.query;
+  const {
+    urlObj,
+    query: { lang },
+  } = req;
+  const searchText = urlObj.searchParams.get('q')?.trim();
   let searchData = null;
-  if (searchText) searchData = await getSearch(req.urlObj.search, lang);
+
+  if (searchText)
+    searchData = await getOrSetCache(searchKey(urlObj), getSearch, urlObj.search, lang);
 
   res.status(200).render('search', {
     data: searchData,
     meta: {
       title: searchText || 'Search',
-      url: req.urlObj,
-      imageUrl: `${req.urlObj.origin}/icon.svg`,
+      url: urlObj,
+      imageUrl: `${urlObj.origin}/icon.svg`,
       description: searchText ? `results for '${searchText}'` : 'search page',
     },
   });

package.json

@@ -34,6 +34,7 @@
     "dotenv": "^16.0.3",
     "express": "^4.18.2",
     "helmet": "^5.1.1",
+    "ioredis": "^5.3.0",
     "morgan": "^1.10.0",
     "pug": "^3.0.2",
     "sass": "^1.57.1"

pnpm-lock.yaml

@@ -13,6 +13,7 @@ specifiers:
   eslint-plugin-import: ^2.26.0
   express: ^4.18.2
   helmet: ^5.1.1
+  ioredis: ^5.3.0
   morgan: ^1.10.0
   nodemon: ^2.0.20
   prettier: ^2.8.2
@@ -26,6 +27,7 @@ dependencies:
   dotenv: 16.0.3
   express: 4.18.2
   helmet: 5.1.1
+  ioredis: 5.3.0
   morgan: 1.10.0
   pug: 3.0.2
   sass: 1.57.1
@@ -122,6 +124,10 @@ packages:
     resolution: {integrity: sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==}
     dev: true
 
+  /@ioredis/commands/1.2.0:
+    resolution: {integrity: sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==}
+    dev: false
+
   /@nodelib/fs.scandir/2.1.5:
     resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==}
     engines: {node: '>= 8'}
@@ -452,6 +458,11 @@ packages:
     optionalDependencies:
       fsevents: 2.3.2
 
+  /cluster-key-slot/1.1.2:
+    resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==}
+    engines: {node: '>=0.10.0'}
+    dev: false
+
   /color-convert/2.0.1:
     resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==}
     engines: {node: '>=7.0.0'}
@@ -595,7 +606,6 @@ packages:
         optional: true
     dependencies:
       ms: 2.1.2
-    dev: true
 
   /deep-is/0.1.4:
     resolution: {integrity: sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==}
@@ -614,6 +624,11 @@ packages:
     engines: {node: '>=0.4.0'}
     dev: false
 
+  /denque/2.1.0:
+    resolution: {integrity: sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==}
+    engines: {node: '>=0.10'}
+    dev: false
+
   /depd/2.0.0:
     resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==}
     engines: {node: '>= 0.8'}
@@ -1320,6 +1335,23 @@ packages:
       side-channel: 1.0.4
     dev: true
 
+  /ioredis/5.3.0:
+    resolution: {integrity: sha512-Id9jKHhsILuIZpHc61QkagfVdUj2Rag5GzG1TGEvRNeM7dtTOjICgjC+tvqYxi//PuX2wjQ+Xjva2ONBuf92Pw==}
+    engines: {node: '>=12.22.0'}
+    dependencies:
+      '@ioredis/commands': 1.2.0
+      cluster-key-slot: 1.1.2
+      debug: 4.3.4
+      denque: 2.1.0
+      lodash.defaults: 4.2.0
+      lodash.isarguments: 3.1.0
+      redis-errors: 1.2.0
+      redis-parser: 3.0.0
+      standard-as-callback: 2.1.0
+    transitivePeerDependencies:
+      - supports-color
+    dev: false
+
   /ipaddr.js/1.9.1:
     resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==}
     engines: {node: '>= 0.10'}
@@ -1512,6 +1544,14 @@ packages:
       p-locate: 5.0.0
     dev: true
 
+  /lodash.defaults/4.2.0:
+    resolution: {integrity: sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==}
+    dev: false
+
+  /lodash.isarguments/3.1.0:
+    resolution: {integrity: sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==}
+    dev: false
+
   /lodash.merge/4.6.2:
     resolution: {integrity: sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==}
     dev: true
@@ -1583,7 +1623,6 @@ packages:
 
   /ms/2.1.2:
     resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==}
-    dev: true
 
   /ms/2.1.3:
     resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==}
@@ -1932,6 +1971,18 @@ packages:
     dependencies:
       picomatch: 2.3.1
 
+  /redis-errors/1.2.0:
+    resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==}
+    engines: {node: '>=4'}
+    dev: false
+
+  /redis-parser/3.0.0:
+    resolution: {integrity: sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==}
+    engines: {node: '>=4'}
+    dependencies:
+      redis-errors: 1.2.0
+    dev: false
+
   /regexp.prototype.flags/1.4.3:
     resolution: {integrity: sha512-fjggEOO3slI6Wvgjwflkc4NFRCTZAu5CnNfBd5qOMYhWdn67nJBBu34/TkD++eeFmd8C9r9jfXJ27+nSiRkSUA==}
     engines: {node: '>= 0.4'}
@@ -2098,6 +2149,10 @@ packages:
     engines: {node: '>=0.10.0'}
     dev: false
 
+  /standard-as-callback/2.1.0:
+    resolution: {integrity: sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==}
+    dev: false
+
   /statuses/2.0.1:
     resolution: {integrity: sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==}
     engines: {node: '>= 0.8'}

utils/cacheKeys.js

@@ -0,0 +1,35 @@
+////////////////////////////////////////////////////////
+//                   LOCAL HELPERS
+////////////////////////////////////////////////////////
+const getLang = urlObj => urlObj.searchParams.get('lang') || 'en';
+const formatSlug = (slug, charToRemove) =>
+  slug.replace(charToRemove, '').toLowerCase();
+
+////////////////////////////////////////////////////////
+//                      EXPORTS
+////////////////////////////////////////////////////////
+export const searchKey = urlObj => {
+  const slug = formatSlug(urlObj.search, '?');
+  return `search:${slug}`;
+};
+
+export const answersKey = urlObj => {
+  const slug = formatSlug(urlObj.pathname, '/');
+  const lang = getLang(urlObj);
+
+  return `answers:${slug}&lang=${lang}`;
+};
+
+export const topicKey = urlObj => {
+  const slug = formatSlug(urlObj.pathname, '/topic/');
+  const lang = getLang(urlObj);
+
+  return `topic:${slug}&lang=${lang}`;
+};
+
+export const profileKey = urlObj => {
+  const slug = formatSlug(urlObj.pathname, '/profile/');
+  const lang = getLang(urlObj);
+
+  return `profile:${slug}&lang=${lang}`;
+};

utils/getOrSetCache.js

@@ -0,0 +1,15 @@
+import redis from './redis.js';
+
+const ttl = process.env.REDIS_TTL || 3600;
+
+const getOrSetCache = async (key, callback, ...callbackArgs) => {
+  const data = await redis.get(key);
+  if (data) return JSON.parse(data);
+
+  const dataToCache = await callback(...callbackArgs);
+  await redis.set(key, JSON.stringify(dataToCache), 'EX', ttl);
+
+  return dataToCache;
+};
+
+export default getOrSetCache;

utils/redis.js

@@ -0,0 +1,12 @@
+/* eslint-disable no-unused-vars */
+import Redis from 'ioredis';
+
+const redisUrl = process.env.REDIS_URL;
+
+const stub = {
+  get: async key => {},
+  set: async (key, value, secondsToken, seconds) => {},
+};
+
+const redis = redisUrl ? new Redis(redisUrl) : stub;
+export default redis;