diff --git a/.env.example b/.env.example index 5ad5899..d2e308f 100644 --- a/.env.example +++ b/.env.example @@ -8,6 +8,9 @@ NODE_ENV=production ## user agent and accept header that quora will see # AXIOS_USER_AGENT='axios/0.26.1' # AXIOS_ACCEPT='application/json, text/plain, */*' +## for caching api responses using redis +# REDIS_URL=localhost:6379 # if left unset, there'll be no caching +# REDIS_TTL=3600 ### for specific use-cases. ## add any value here (e.g.: 1, true, 'por favor') if you're using any service where http is the preferred method(e.g.: tor, i2p). else leave it blank diff --git a/.gitignore b/.gitignore index 36d4baf..f0abbf1 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ node_modules/* .env dev-data/* -public/css/* \ No newline at end of file +public/css/* +dump.rdb \ No newline at end of file diff --git a/README.md b/README.md index e4ec825..742c619 100644 --- a/README.md +++ b/README.md @@ -188,7 +188,7 @@ From [their privacy policy](https://www.quora.com/about/privacy) ## To-Do - [x] add missing routes like topics, profile, and search -- [ ] use redis +- [x] use redis - [x] serve images and other assets from Quetre - [x] implement a better installation method - [ ] implement other trivial routes like a specific answer, spaces, etc. @@ -200,7 +200,7 @@ From [their privacy policy](https://www.quora.com/about/privacy) ### Manual -1. Install [Node.js](https://nodejs.org/en/) and [Git](https://git-scm.com/). Instructions are on their websites. +1. Install [Node.js](https://nodejs.org/en/), [Git](https://git-scm.com/), and [Redis](https://redis.io)(Optional). Instructions are on their websites. 2. Clone and set up the repository. @@ -211,6 +211,8 @@ From [their privacy policy](https://www.quora.com/about/privacy) # change `pnpm` to `npm run` here as well as in package.json if you use `npm` pnpm install pnpm start + # optional + redis-server # useful for caching api responses ``` Quetre will start running at http://localhost:3000. diff --git a/controllers/apiController.js b/controllers/apiController.js index c301ad3..f4a3f0a 100644 --- a/controllers/apiController.js +++ b/controllers/apiController.js @@ -8,6 +8,8 @@ import getAnswers from '../fetchers/getAnswers.js'; import getTopic from '../fetchers/getTopic.js'; import getProfile from '../fetchers/getProfile.js'; import getSearch from '../fetchers/getSearch.js'; +import getOrSetCache from '../utils/getOrSetCache.js'; +import { answersKey, profileKey, searchKey, topicKey } from '../utils/cacheKeys.js'; //////////////////////////////////////////////////////// // EXPORTS @@ -21,31 +23,49 @@ export const about = (req, res, next) => { }; export const answers = catchAsyncErrors(async (req, res, next) => { - const { slug } = req.params; - const { lang } = req.query; + const { + urlObj, + params: { slug }, + query: { lang }, + } = req; - const data = await getAnswers(slug, lang); + const data = await getOrSetCache(answersKey(urlObj), getAnswers, slug, lang); res.status(200).json({ status: 'success', data }); }); export const topic = catchAsyncErrors(async (req, res, next) => { - const { slug } = req.params; - const { lang } = req.query; + const { + urlObj, + params: { slug }, + query: { lang }, + } = req; - const data = await getTopic(slug, lang); + const data = await getOrSetCache(topicKey(urlObj), getTopic, slug, lang); res.status(200).json({ status: 'success', data }); }); export const profile = catchAsyncErrors(async (req, res, next) => { - const data = await getProfile(req.params.name); + const { + urlObj, + params: { name }, + query: { lang }, + } = req; + + const data = await getOrSetCache(profileKey(urlObj), getProfile, name, lang); res.status(200).json({ status: 'success', data }); }); export const search = catchAsyncErrors(async (req, res, next) => { - const searchText = req.urlObj.searchParams.get('q')?.trim(); // no search to perform if there isn't any query + const { + urlObj, + query: { lang }, + } = req; + const searchText = urlObj.searchParams.get('q')?.trim(); // no search to perform if there isn't any query let searchData = null; - if (searchText) searchData = await getSearch(req.urlObj.search); + + if (searchText) + searchData = await getOrSetCache(searchKey(urlObj), getSearch, urlObj.search, lang); res.status(200).json({ status: 'success', data: searchData }); }); @@ -72,5 +92,6 @@ export const image = catchAsyncErrors(async (req, res, next) => { const imageRes = await axiosInstance.get(path, { responseType: 'stream' }); res.set('Content-Type', imageRes.headers['content-type']); + res.set('Cache-Control', 'public, max-age=315360000'); return imageRes.data.pipe(res); }); diff --git a/controllers/viewController.js b/controllers/viewController.js index cc531fe..af8c081 100644 --- a/controllers/viewController.js +++ b/controllers/viewController.js @@ -8,6 +8,8 @@ import getTopic from '../fetchers/getTopic.js'; import { acceptedLanguages, nonSlugRoutes } from '../utils/constants.js'; import getProfile from '../fetchers/getProfile.js'; import getSearch from '../fetchers/getSearch.js'; +import getOrSetCache from '../utils/getOrSetCache.js'; +import { answersKey, profileKey, searchKey, topicKey } from '../utils/cacheKeys.js'; //////////////////////////////////////////////////////// // EXPORTS @@ -36,16 +38,17 @@ export const privacy = (req, res, next) => { }; export const answers = catchAsyncErrors(async (req, res, next) => { - const { slug } = req.params; - const { lang } = req.query; + const { + urlObj, + params: { slug }, + query: { lang }, + } = req; // added this so that a request by browser to get favicon doesn't end up being interpreted as a slug if (nonSlugRoutes.includes(slug)) return next(); - const answersData = await getAnswers(slug, lang); - const title = answersData.question.text[0].spans - .map(span => span.text) - .join(''); + const answersData = await getOrSetCache(answersKey(urlObj), getAnswers, slug, lang); + const title = answersData.question.text[0].spans.map(span => span.text).join(''); return res.status(200).render('answers', { data: answersData, @@ -59,50 +62,62 @@ export const answers = catchAsyncErrors(async (req, res, next) => { }); export const topic = catchAsyncErrors(async (req, res, next) => { - const { slug } = req.params; - const { lang } = req.query; + const { + urlObj, + params: { slug }, + query: { lang }, + } = req; - const topicData = await getTopic(slug, lang); + const topicData = await getOrSetCache(topicKey(urlObj), getTopic, slug, lang); res.status(200).render('topic', { data: topicData, meta: { title: topicData.name, - url: req.urlObj, - imageUrl: `${req.urlObj.origin}/icon.svg`, + url: urlObj, + imageUrl: `${urlObj.origin}/icon.svg`, description: `Information about ${topicData.name} topic.`, }, }); }); export const profile = catchAsyncErrors(async (req, res, next) => { - const { name } = req.params; - const { lang } = req.query; - const profileData = await getProfile(name, lang); + const { + urlObj, + params: { name }, + query: { lang }, + } = req; + + const profileData = await getOrSetCache(profileKey(urlObj), getProfile, name, lang); res.status(200).render('profile', { data: profileData, meta: { title: profileData.basic.name, - url: req.urlObj, - imageUrl: `${req.urlObj.origin}/icon.svg`, + url: urlObj, + imageUrl: `${urlObj.origin}/icon.svg`, description: `${profileData.basic.name}'s profile.`, }, }); }); export const search = catchAsyncErrors(async (req, res, next) => { - const searchText = req.urlObj.searchParams.get('q')?.trim(); - const { lang } = req.query; + const { + urlObj, + query: { lang }, + } = req; + const searchText = urlObj.searchParams.get('q')?.trim(); let searchData = null; - if (searchText) searchData = await getSearch(req.urlObj.search, lang); + + if (searchText) + searchData = await getOrSetCache(searchKey(urlObj), getSearch, urlObj.search, lang); res.status(200).render('search', { data: searchData, meta: { title: searchText || 'Search', - url: req.urlObj, - imageUrl: `${req.urlObj.origin}/icon.svg`, + url: urlObj, + imageUrl: `${urlObj.origin}/icon.svg`, description: searchText ? `results for '${searchText}'` : 'search page', }, }); diff --git a/package.json b/package.json index 2c67396..1a11273 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "dotenv": "^16.0.3", "express": "^4.18.2", "helmet": "^5.1.1", + "ioredis": "^5.3.0", "morgan": "^1.10.0", "pug": "^3.0.2", "sass": "^1.57.1" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 89451fb..7939cba 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -13,6 +13,7 @@ specifiers: eslint-plugin-import: ^2.26.0 express: ^4.18.2 helmet: ^5.1.1 + ioredis: ^5.3.0 morgan: ^1.10.0 nodemon: ^2.0.20 prettier: ^2.8.2 @@ -26,6 +27,7 @@ dependencies: dotenv: 16.0.3 express: 4.18.2 helmet: 5.1.1 + ioredis: 5.3.0 morgan: 1.10.0 pug: 3.0.2 sass: 1.57.1 @@ -122,6 +124,10 @@ packages: resolution: {integrity: sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==} dev: true + /@ioredis/commands/1.2.0: + resolution: {integrity: sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==} + dev: false + /@nodelib/fs.scandir/2.1.5: resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==} engines: {node: '>= 8'} @@ -452,6 +458,11 @@ packages: optionalDependencies: fsevents: 2.3.2 + /cluster-key-slot/1.1.2: + resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==} + engines: {node: '>=0.10.0'} + dev: false + /color-convert/2.0.1: resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} engines: {node: '>=7.0.0'} @@ -595,7 +606,6 @@ packages: optional: true dependencies: ms: 2.1.2 - dev: true /deep-is/0.1.4: resolution: {integrity: sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==} @@ -614,6 +624,11 @@ packages: engines: {node: '>=0.4.0'} dev: false + /denque/2.1.0: + resolution: {integrity: sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==} + engines: {node: '>=0.10'} + dev: false + /depd/2.0.0: resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} engines: {node: '>= 0.8'} @@ -1320,6 +1335,23 @@ packages: side-channel: 1.0.4 dev: true + /ioredis/5.3.0: + resolution: {integrity: sha512-Id9jKHhsILuIZpHc61QkagfVdUj2Rag5GzG1TGEvRNeM7dtTOjICgjC+tvqYxi//PuX2wjQ+Xjva2ONBuf92Pw==} + engines: {node: '>=12.22.0'} + dependencies: + '@ioredis/commands': 1.2.0 + cluster-key-slot: 1.1.2 + debug: 4.3.4 + denque: 2.1.0 + lodash.defaults: 4.2.0 + lodash.isarguments: 3.1.0 + redis-errors: 1.2.0 + redis-parser: 3.0.0 + standard-as-callback: 2.1.0 + transitivePeerDependencies: + - supports-color + dev: false + /ipaddr.js/1.9.1: resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==} engines: {node: '>= 0.10'} @@ -1512,6 +1544,14 @@ packages: p-locate: 5.0.0 dev: true + /lodash.defaults/4.2.0: + resolution: {integrity: sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==} + dev: false + + /lodash.isarguments/3.1.0: + resolution: {integrity: sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==} + dev: false + /lodash.merge/4.6.2: resolution: {integrity: sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==} dev: true @@ -1583,7 +1623,6 @@ packages: /ms/2.1.2: resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==} - dev: true /ms/2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} @@ -1932,6 +1971,18 @@ packages: dependencies: picomatch: 2.3.1 + /redis-errors/1.2.0: + resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==} + engines: {node: '>=4'} + dev: false + + /redis-parser/3.0.0: + resolution: {integrity: sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==} + engines: {node: '>=4'} + dependencies: + redis-errors: 1.2.0 + dev: false + /regexp.prototype.flags/1.4.3: resolution: {integrity: sha512-fjggEOO3slI6Wvgjwflkc4NFRCTZAu5CnNfBd5qOMYhWdn67nJBBu34/TkD++eeFmd8C9r9jfXJ27+nSiRkSUA==} engines: {node: '>= 0.4'} @@ -2098,6 +2149,10 @@ packages: engines: {node: '>=0.10.0'} dev: false + /standard-as-callback/2.1.0: + resolution: {integrity: sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==} + dev: false + /statuses/2.0.1: resolution: {integrity: sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==} engines: {node: '>= 0.8'} diff --git a/utils/cacheKeys.js b/utils/cacheKeys.js new file mode 100644 index 0000000..5ec4c65 --- /dev/null +++ b/utils/cacheKeys.js @@ -0,0 +1,35 @@ +//////////////////////////////////////////////////////// +// LOCAL HELPERS +//////////////////////////////////////////////////////// +const getLang = urlObj => urlObj.searchParams.get('lang') || 'en'; +const formatSlug = (slug, charToRemove) => + slug.replace(charToRemove, '').toLowerCase(); + +//////////////////////////////////////////////////////// +// EXPORTS +//////////////////////////////////////////////////////// +export const searchKey = urlObj => { + const slug = formatSlug(urlObj.search, '?'); + return `search:${slug}`; +}; + +export const answersKey = urlObj => { + const slug = formatSlug(urlObj.pathname, '/'); + const lang = getLang(urlObj); + + return `answers:${slug}&lang=${lang}`; +}; + +export const topicKey = urlObj => { + const slug = formatSlug(urlObj.pathname, '/topic/'); + const lang = getLang(urlObj); + + return `topic:${slug}&lang=${lang}`; +}; + +export const profileKey = urlObj => { + const slug = formatSlug(urlObj.pathname, '/profile/'); + const lang = getLang(urlObj); + + return `profile:${slug}&lang=${lang}`; +}; diff --git a/utils/getOrSetCache.js b/utils/getOrSetCache.js new file mode 100644 index 0000000..35cccc8 --- /dev/null +++ b/utils/getOrSetCache.js @@ -0,0 +1,15 @@ +import redis from './redis.js'; + +const ttl = process.env.REDIS_TTL || 3600; + +const getOrSetCache = async (key, callback, ...callbackArgs) => { + const data = await redis.get(key); + if (data) return JSON.parse(data); + + const dataToCache = await callback(...callbackArgs); + await redis.set(key, JSON.stringify(dataToCache), 'EX', ttl); + + return dataToCache; +}; + +export default getOrSetCache; diff --git a/utils/redis.js b/utils/redis.js new file mode 100644 index 0000000..e49c1f3 --- /dev/null +++ b/utils/redis.js @@ -0,0 +1,12 @@ +/* eslint-disable no-unused-vars */ +import Redis from 'ioredis'; + +const redisUrl = process.env.REDIS_URL; + +const stub = { + get: async key => {}, + set: async (key, value, secondsToken, seconds) => {}, +}; + +const redis = redisUrl ? new Redis(redisUrl) : stub; +export default redis;