feat: render route

.env.example

@@ -0,0 +1,5 @@
+HOST="0.0.0.0"
+PORT=8080
+TIMEOUT=0
+WAIT_FOR_PAGE_LOAD_TIMEOUT=2000
+REVERSE_PROXY=false

.eslintrc.json

@@ -0,0 +1,14 @@
+{
+  "env": {
+    "browser": true,
+    "es2021": true
+  },
+  "extends": ["eslint:recommended", "plugin:@typescript-eslint/recommended"],
+  "parser": "@typescript-eslint/parser",
+  "parserOptions": {
+    "ecmaVersion": "latest",
+    "sourceType": "module"
+  },
+  "plugins": ["@typescript-eslint"],
+  "rules": {}
+}

.github/workflows/build.yml

@@ -0,0 +1,19 @@
+name: build-check
+on: pull_request
+jobs:
+  build-check:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v2
+        with:
+          node-version: '18'
+
+      - name: Install dependencies
+        run: npm install
+
+      - name: Start build
+        run: npm run build

.github/workflows/deploy.yml

@@ -0,0 +1,49 @@
+#
+name: Create and publish a Docker image
+
+# Configures this workflow to run every time a change is pushed to the branch called `release`.
+on:
+  release:
+    types:
+      - published
+
+# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
+    permissions:
+      contents: read
+      packages: write
+      #
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+      # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels.
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+      # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
+      # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository.
+      # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.
+      - name: Build and push Docker image
+        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/format-check.yml

@@ -0,0 +1,19 @@
+name: format-check
+on: pull_request
+jobs:
+  format-check:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v2
+        with:
+          node-version: '18'
+
+      - name: Install dependencies
+        run: npm install
+
+      - name: Check formatting
+        run: npm run format:check

.gitignore

@@ -1,78 +1,99 @@
 # Logs
+
 logs
-*.log
-npm-debug.log*
+_.log
+npm-debug.log_
 yarn-debug.log*
 yarn-error.log*
 lerna-debug.log*
 .pnpm-debug.log*
 
 # Diagnostic reports (https://nodejs.org/api/report.html)
-report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
+
+report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
 
 # Runtime data
+
 pids
-*.pid
-*.seed
-*.pid.lock
+_.pid
+_.seed
+\*.pid.lock
 
 # Directory for instrumented libs generated by jscoverage/JSCover
+
 lib-cov
 
 # Coverage directory used by tools like istanbul
+
 coverage
-*.lcov
+\*.lcov
 
 # nyc test coverage
+
 .nyc_output
 
 # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+
 .grunt
 
 # Bower dependency directory (https://bower.io/)
+
 bower_components
 
 # node-waf configuration
+
 .lock-wscript
 
 # Compiled binary addons (https://nodejs.org/api/addons.html)
+
 build/Release
 
 # Dependency directories
+
 node_modules/
 jspm_packages/
 
 # Snowpack dependency directory (https://snowpack.dev/)
+
 web_modules/
 
 # TypeScript cache
-*.tsbuildinfo
+
+\*.tsbuildinfo
 
 # Optional npm cache directory
+
 .npm
 
 # Optional eslint cache
+
 .eslintcache
 
 # Optional stylelint cache
+
 .stylelintcache
 
 # Microbundle cache
+
 .rpt2_cache/
 .rts2_cache_cjs/
 .rts2_cache_es/
 .rts2_cache_umd/
 
 # Optional REPL history
+
 .node_repl_history
 
 # Output of 'npm pack'
-*.tgz
+
+\*.tgz
 
 # Yarn Integrity file
+
 .yarn-integrity
 
 # dotenv environment variable files
+
 .env
 .env.development.local
 .env.test.local
@@ -80,48 +101,64 @@ web_modules/
 .env.local
 
 # parcel-bundler cache (https://parceljs.org/)
+
 .cache
 .parcel-cache
 
 # Next.js build output
+
 .next
 out
 
 # Nuxt.js build / generate output
+
 .nuxt
 dist
 
 # Gatsby files
+
 .cache/
+
 # Comment in the public line in if your project uses Gatsby and not Next.js
+
 # https://nextjs.org/blog/next-9-1#public-directory-support
+
 # public
 
 # vuepress build output
+
 .vuepress/dist
 
 # vuepress v2.x temp and cache directory
+
 .temp
 .cache
 
 # Serverless directories
+
 .serverless/
 
 # FuseBox cache
+
 .fusebox/
 
 # DynamoDB Local files
+
 .dynamodb/
 
 # TernJS port file
+
 .tern-port
 
 # Stores VSCode versions used for testing VSCode extensions
+
 .vscode-test
 
 # yarn v2
+
 .yarn/cache
 .yarn/unplugged
 .yarn/build-state.yml
 .yarn/install-state.gz
-.pnp.*
+.pnp.\*
+

.prettierrc

@@ -0,0 +1,5 @@
+{
+  "trailingComma": "es5",
+  "tabWidth": 2,
+  "singleQuote": true
+}

Dockerfile

@@ -0,0 +1,12 @@
+FROM node:18-alpine as build
+WORKDIR /app
+COPY . .
+RUN npm install
+RUN npm run build
+
+FROM node:18-alpine as run
+WORKDIR /app
+COPY --from=build /app/dist/ /app/package*.json ./
+RUN npm install --omit=dev
+CMD npm run start:docker
+EXPOSE 8080

README.md

@@ -1,2 +1,21 @@
 # webder
- Http/s proxy that render pages with js and returns HTML
+
+Http/s proxy that render pages with js and returns HTML
+
+> [!WARNING]
+> This service is not purify html and not blocking XSS attacks. Also it is not block requests to localhost, etc.
+> This is not intended to be used as a user-facing browser. Use it ONLY for making api calls from other services, make sure to run it behind a firewall and use a reverse proxy in front of it.
+
+## Features
+
+- Render pages with js (/render?url=...)
+
+## Usage
+
+For env variables look to `.env.example`. Docker also supported.
+
+```bash
+npm install
+npm run build
+npm start
+```

docker-compose.yml

@@ -0,0 +1,10 @@
+version: "3"
+
+services:
+  txtdot:
+    image: ghcr.io/txtdot/webder:latest
+    ports:
+      - "8080:8080"
+    restart: unless-stopped
+    volumes:
+      - ".env:/app/.env"

package.json

@@ -0,0 +1,43 @@
+{
+  "name": "webder",
+  "version": "1.0.0",
+  "description": "Http/s proxy that render pages with js and returns HTML",
+  "main": "dist/app.js",
+  "private": true,
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "build": "npm run lint && copyfiles .env ./dist/ && tsc",
+    "dev": "tsc-watch --onSuccess \"node ./dist/app.js\"",
+    "start": "cd dist && node ./app.js",
+    "start:docker": "node ./app.js",
+    "format": "prettier --write .",
+    "format:check": "prettier --check .",
+    "lint": "eslint \"src/**\""
+  },
+  "keywords": [],
+  "authors": [
+    "Artemy Egorov <me@artegoser.ru> https://github.com/artegoser",
+    "DarkCat09 <darkcat09@vivaldi.net> https://dc09.ru/",
+    "megahomyak https://github.com/megahomyak"
+  ],
+  "license": "MIT",
+  "devDependencies": {
+    "@types/dompurify": "^3.0.5",
+    "@types/node": "^20.11.30",
+    "@typescript-eslint/eslint-plugin": "^7.3.1",
+    "@typescript-eslint/parser": "^7.3.1",
+    "copyfiles": "^2.4.1",
+    "eslint": "^8.57.0",
+    "tsc-watch": "^6.0.4",
+    "typescript": "^5.4.3"
+  },
+  "dependencies": {
+    "dotenv": "^16.4.5",
+    "fastify": "^4.26.2",
+    "prettier": "^3.2.5",
+    "puppeteer": "^22.6.0",
+    "puppeteer-extra": "^3.3.6",
+    "puppeteer-extra-plugin-adblocker": "^2.13.6",
+    "puppeteer-extra-plugin-stealth": "^2.11.2"
+  }
+}

src/app.ts

@@ -0,0 +1,65 @@
+import { ConfigService } from "./config/config.service";
+import Fastify from "fastify";
+import getConfig from "./config/main";
+import { puppeteer } from "./puppeteer";
+
+class App {
+  config: ConfigService;
+  constructor() {
+    this.config = getConfig();
+  }
+  listen() {
+    const fastify = Fastify({
+      logger: true,
+      trustProxy: this.config.reverse_proxy,
+      connectionTimeout: this.config.timeout,
+    });
+
+    fastify.get<{ Querystring: { url: string } }>(
+      "/render",
+      {
+        schema: {
+          querystring: {
+            type: "object",
+            properties: {
+              url: { type: "string" },
+            },
+            required: ["url"],
+          },
+        },
+      },
+      async (request, reply) => {
+        const browser = await puppeteer.launch();
+        const page = await browser.newPage();
+
+        await page.goto(request.query.url);
+
+        try {
+          await page.waitForNetworkIdle({
+            timeout: this.config.wait_for_page_load_timeout,
+            idleTime: 100,
+          });
+        } catch {
+          reply.header("x-message", "Page load timeout");
+        }
+
+        const data = await page.content();
+
+        browser.close();
+
+        reply.type("text/html");
+        return data;
+      }
+    );
+
+    fastify.listen(
+      { host: this.config.host, port: this.config.port },
+      (err) => {
+        err && console.log(err);
+      }
+    );
+  }
+}
+
+const app = new App();
+app.listen();

src/config/config.service.ts

@@ -0,0 +1,26 @@
+import { config } from "dotenv";
+
+export class ConfigService {
+  public readonly host: string;
+  public readonly port: number;
+  public readonly timeout: number;
+  public readonly wait_for_page_load_timeout: number;
+  public readonly reverse_proxy: boolean;
+  constructor() {
+    config();
+
+    this.host = process.env.HOST || "0.0.0.0";
+    this.port = Number(process.env.PORT) || 8080;
+
+    this.timeout = Number(process.env.TIMEOUT) || 0;
+    this.wait_for_page_load_timeout =
+      Number(process.env.WAIT_FOR_PAGE_LOAD_TIMEOUT) || 2000;
+
+    this.reverse_proxy = this.parseBool(process.env.REVERSE_PROXY, false);
+  }
+
+  parseBool(value: string | undefined, def: boolean): boolean {
+    if (!value) return def;
+    return value === "true" || value === "1";
+  }
+}

src/config/main.ts

@@ -0,0 +1,12 @@
+import { ConfigService } from "./config.service";
+
+let configSvc: ConfigService | undefined;
+
+export default function getConfig(): ConfigService {
+  if (configSvc) {
+    return configSvc;
+  }
+
+  configSvc = new ConfigService();
+  return configSvc;
+}

src/puppeteer.ts

@@ -0,0 +1,16 @@
+import puppeteer from 'puppeteer-extra';
+import { DEFAULT_INTERCEPT_RESOLUTION_PRIORITY } from 'puppeteer';
+import AdblockerPlugin from 'puppeteer-extra-plugin-adblocker';
+import StealthPlugin from 'puppeteer-extra-plugin-stealth';
+
+puppeteer.use(StealthPlugin());
+
+puppeteer.use(
+  AdblockerPlugin({
+    interceptResolutionPriority: DEFAULT_INTERCEPT_RESOLUTION_PRIORITY,
+    blockTrackers: true,
+    blockTrackersAndAnnoyances: true,
+  })
+);
+
+export { puppeteer };

tsconfig.json

@@ -0,0 +1,109 @@
+{
+  "compilerOptions": {
+    /* Visit https://aka.ms/tsconfig to read more about this file */
+
+    /* Projects */
+    // "incremental": true,                              /* Save .tsbuildinfo files to allow for incremental compilation of projects. */
+    // "composite": true,                                /* Enable constraints that allow a TypeScript project to be used with project references. */
+    // "tsBuildInfoFile": "./.tsbuildinfo",              /* Specify the path to .tsbuildinfo incremental compilation file. */
+    // "disableSourceOfProjectReferenceRedirect": true,  /* Disable preferring source files instead of declaration files when referencing composite projects. */
+    // "disableSolutionSearching": true,                 /* Opt a project out of multi-project reference checking when editing. */
+    // "disableReferencedProjectLoad": true,             /* Reduce the number of projects loaded automatically by TypeScript. */
+
+    /* Language and Environment */
+    "target": "es2016" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
+    // "lib": [],                                        /* Specify a set of bundled library declaration files that describe the target runtime environment. */
+    // "jsx": "preserve",                                /* Specify what JSX code is generated. */
+    // "experimentalDecorators": true,                   /* Enable experimental support for legacy experimental decorators. */
+    // "emitDecoratorMetadata": true,                    /* Emit design-type metadata for decorated declarations in source files. */
+    // "jsxFactory": "",                                 /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */
+    // "jsxFragmentFactory": "",                         /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */
+    // "jsxImportSource": "",                            /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */
+    // "reactNamespace": "",                             /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */
+    // "noLib": true,                                    /* Disable including any library files, including the default lib.d.ts. */
+    // "useDefineForClassFields": true,                  /* Emit ECMAScript-standard-compliant class fields. */
+    // "moduleDetection": "auto",                        /* Control what method is used to detect module-format JS files. */
+
+    /* Modules */
+    "module": "commonjs" /* Specify what module code is generated. */,
+    // "rootDir": "./",                                  /* Specify the root folder within your source files. */
+    // "moduleResolution": "node10",                     /* Specify how TypeScript looks up a file from a given module specifier. */
+    // "baseUrl": "./",                                  /* Specify the base directory to resolve non-relative module names. */
+    // "paths": {},                                      /* Specify a set of entries that re-map imports to additional lookup locations. */
+    // "rootDirs": [],                                   /* Allow multiple folders to be treated as one when resolving modules. */
+    // "typeRoots": [],                                  /* Specify multiple folders that act like './node_modules/@types'. */
+    // "types": [],                                      /* Specify type package names to be included without being referenced in a source file. */
+    // "allowUmdGlobalAccess": true,                     /* Allow accessing UMD globals from modules. */
+    // "moduleSuffixes": [],                             /* List of file name suffixes to search when resolving a module. */
+    // "allowImportingTsExtensions": true,               /* Allow imports to include TypeScript file extensions. Requires '--moduleResolution bundler' and either '--noEmit' or '--emitDeclarationOnly' to be set. */
+    // "resolvePackageJsonExports": true,                /* Use the package.json 'exports' field when resolving package imports. */
+    // "resolvePackageJsonImports": true,                /* Use the package.json 'imports' field when resolving imports. */
+    // "customConditions": [],                           /* Conditions to set in addition to the resolver-specific defaults when resolving imports. */
+    // "resolveJsonModule": true,                        /* Enable importing .json files. */
+    // "allowArbitraryExtensions": true,                 /* Enable importing files with any extension, provided a declaration file is present. */
+    // "noResolve": true,                                /* Disallow 'import's, 'require's or '<reference>'s from expanding the number of files TypeScript should add to a project. */
+
+    /* JavaScript Support */
+    // "allowJs": true,                                  /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */
+    // "checkJs": true,                                  /* Enable error reporting in type-checked JavaScript files. */
+    // "maxNodeModuleJsDepth": 1,                        /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */
+
+    /* Emit */
+    // "declaration": true,                              /* Generate .d.ts files from TypeScript and JavaScript files in your project. */
+    // "declarationMap": true,                           /* Create sourcemaps for d.ts files. */
+    // "emitDeclarationOnly": true,                      /* Only output d.ts files and not JavaScript files. */
+    // "sourceMap": true,                                /* Create source map files for emitted JavaScript files. */
+    // "inlineSourceMap": true,                          /* Include sourcemap files inside the emitted JavaScript. */
+    // "outFile": "./",                                  /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */
+    "outDir": "./dist/" /* Specify an output folder for all emitted files. */,
+    // "removeComments": true,                           /* Disable emitting comments. */
+    // "noEmit": true,                                   /* Disable emitting files from a compilation. */
+    // "importHelpers": true,                            /* Allow importing helper functions from tslib once per project, instead of including them per-file. */
+    // "importsNotUsedAsValues": "remove",               /* Specify emit/checking behavior for imports that are only used for types. */
+    // "downlevelIteration": true,                       /* Emit more compliant, but verbose and less performant JavaScript for iteration. */
+    // "sourceRoot": "",                                 /* Specify the root path for debuggers to find the reference source code. */
+    // "mapRoot": "",                                    /* Specify the location where debugger should locate map files instead of generated locations. */
+    // "inlineSources": true,                            /* Include source code in the sourcemaps inside the emitted JavaScript. */
+    // "emitBOM": true,                                  /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */
+    // "newLine": "crlf",                                /* Set the newline character for emitting files. */
+    // "stripInternal": true,                            /* Disable emitting declarations that have '@internal' in their JSDoc comments. */
+    // "noEmitHelpers": true,                            /* Disable generating custom helper functions like '__extends' in compiled output. */
+    // "noEmitOnError": true,                            /* Disable emitting files if any type checking errors are reported. */
+    // "preserveConstEnums": true,                       /* Disable erasing 'const enum' declarations in generated code. */
+    // "declarationDir": "./",                           /* Specify the output directory for generated declaration files. */
+    // "preserveValueImports": true,                     /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */
+
+    /* Interop Constraints */
+    // "isolatedModules": true,                          /* Ensure that each file can be safely transpiled without relying on other imports. */
+    // "verbatimModuleSyntax": true,                     /* Do not transform or elide any imports or exports not marked as type-only, ensuring they are written in the output file's format based on the 'module' setting. */
+    // "allowSyntheticDefaultImports": true,             /* Allow 'import x from y' when a module doesn't have a default export. */
+    "esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */,
+    // "preserveSymlinks": true,                         /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */
+    "forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */,
+
+    /* Type Checking */
+    "strict": true /* Enable all strict type-checking options. */,
+    // "noImplicitAny": true,                            /* Enable error reporting for expressions and declarations with an implied 'any' type. */
+    // "strictNullChecks": true,                         /* When type checking, take into account 'null' and 'undefined'. */
+    // "strictFunctionTypes": true,                      /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */
+    // "strictBindCallApply": true,                      /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */
+    // "strictPropertyInitialization": true,             /* Check for class properties that are declared but not set in the constructor. */
+    // "noImplicitThis": true,                           /* Enable error reporting when 'this' is given the type 'any'. */
+    // "useUnknownInCatchVariables": true,               /* Default catch clause variables as 'unknown' instead of 'any'. */
+    // "alwaysStrict": true,                             /* Ensure 'use strict' is always emitted. */
+    // "noUnusedLocals": true,                           /* Enable error reporting when local variables aren't read. */
+    // "noUnusedParameters": true,                       /* Raise an error when a function parameter isn't read. */
+    // "exactOptionalPropertyTypes": true,               /* Interpret optional property types as written, rather than adding 'undefined'. */
+    // "noImplicitReturns": true,                        /* Enable error reporting for codepaths that do not explicitly return in a function. */
+    // "noFallthroughCasesInSwitch": true,               /* Enable error reporting for fallthrough cases in switch statements. */
+    // "noUncheckedIndexedAccess": true,                 /* Add 'undefined' to a type when accessed using an index. */
+    // "noImplicitOverride": true,                       /* Ensure overriding members in derived classes are marked with an override modifier. */
+    // "noPropertyAccessFromIndexSignature": true,       /* Enforces using indexed accessors for keys declared using an indexed type. */
+    // "allowUnusedLabels": true,                        /* Disable error reporting for unused labels. */
+    // "allowUnreachableCode": true,                     /* Disable error reporting for unreachable code. */
+
+    /* Completeness */
+    // "skipDefaultLibCheck": true,                      /* Skip type checking .d.ts files that are included with TypeScript. */
+    "skipLibCheck": true /* Skip type checking all .d.ts files. */
+  }
+}