2023-08-30 09:25:55 +00:00
<!doctype html>
< html lang = "en" class = "no-js" >
< head >
< meta charset = "utf-8" >
< meta name = "viewport" content = "width=device-width,initial-scale=1" >
2024-05-27 09:46:55 +00:00
< link rel = "next" href = "selfhost/" >
2023-08-31 07:14:19 +00:00
2024-05-27 09:38:37 +00:00
< link rel = "icon" href = "assets/images/favicon.png" >
2023-08-30 09:25:55 +00:00
< meta name = "generator" content = "mkdocs-1.5.2, mkdocs-material-9.2.5" >
2024-05-27 09:38:37 +00:00
< title > txtdot< / title >
2023-08-30 09:25:55 +00:00
2024-05-27 09:38:37 +00:00
< link rel = "stylesheet" href = "assets/stylesheets/main.0e669242.min.css" >
2023-08-30 09:25:55 +00:00
2024-05-27 09:38:37 +00:00
< link rel = "stylesheet" href = "assets/stylesheets/palette.85d0ee34.min.css" >
2023-08-30 09:25:55 +00:00
< link rel = "preconnect" href = "https://fonts.gstatic.com" crossorigin >
< link rel = "stylesheet" href = "https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback" >
< style > : root { --md-text-font : "Roboto" ; --md-code-font : "Roboto Mono" } < / style >
2024-05-27 09:38:37 +00:00
< script > _ _md _scope = new URL ( "." , location ) , _ _md _hash = e => [ ... e ] . reduce ( ( e , _ ) => ( e << 5 ) - e + _ . charCodeAt ( 0 ) , 0 ) , _ _md _get = ( e , _ = localStorage , t = _ _md _scope ) => JSON . parse ( _ . getItem ( t . pathname + "." + e ) ) , _ _md _set = ( e , _ , t = localStorage , a = _ _md _scope ) => { try { t . setItem ( a . pathname + "." + e , JSON . stringify ( _ ) ) } catch ( e ) { } } < / script >
2023-08-30 09:25:55 +00:00
< / head >
< body dir = "ltr" data-md-color-scheme = "default" data-md-color-primary = "indigo" data-md-color-accent = "indigo" >
< script > var palette = _ _md _get ( "__palette" ) ; if ( palette && "object" == typeof palette . color ) for ( var key of Object . keys ( palette . color ) ) document . body . setAttribute ( "data-md-color-" + key , palette . color [ key ] ) < / script >
< input class = "md-toggle" data-md-toggle = "drawer" type = "checkbox" id = "__drawer" autocomplete = "off" >
< input class = "md-toggle" data-md-toggle = "search" type = "checkbox" id = "__search" autocomplete = "off" >
< label class = "md-overlay" for = "__drawer" > < / label >
< div data-md-component = "skip" >
< a href = "#getting-started" class = "md-skip" >
Skip to content
< / a >
< / div >
< div data-md-component = "announce" >
< / div >
< header class = "md-header md-header--shadow" data-md-component = "header" >
< nav class = "md-header__inner md-grid" aria-label = "Header" >
2024-05-27 09:38:37 +00:00
< a href = "." title = "txtdot" class = "md-header__button md-logo" aria-label = "txtdot" data-md-component = "logo" >
2023-08-30 09:25:55 +00:00
< svg xmlns = "http://www.w3.org/2000/svg" viewBox = "0 0 24 24" > < path d = "M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54Z" / > < / svg >
< / a >
< label class = "md-header__button md-icon" for = "__drawer" >
< svg xmlns = "http://www.w3.org/2000/svg" viewBox = "0 0 24 24" > < path d = "M3 6h18v2H3V6m0 5h18v2H3v-2m0 5h18v2H3v-2Z" / > < / svg >
< / label >
< div class = "md-header__title" data-md-component = "header-title" >
< div class = "md-header__ellipsis" >
< div class = "md-header__topic" >
< span class = "md-ellipsis" >
txtdot
< / span >
< / div >
< div class = "md-header__topic" data-md-component = "header-topic" >
< span class = "md-ellipsis" >
Getting Started
< / span >
< / div >
< / div >
< / div >
< form class = "md-header__option" data-md-component = "palette" >
< input class = "md-option" data-md-color-media = "(prefers-color-scheme: light)" data-md-color-scheme = "default" data-md-color-primary = "indigo" data-md-color-accent = "indigo" aria-label = "Switch to dark mode" type = "radio" name = "__palette" id = "__palette_1" >
< label class = "md-header__button md-icon" title = "Switch to dark mode" for = "__palette_2" hidden >
< svg xmlns = "http://www.w3.org/2000/svg" viewBox = "0 0 24 24" > < path d = "M12 8a4 4 0 0 0-4 4 4 4 0 0 0 4 4 4 4 0 0 0 4-4 4 4 0 0 0-4-4m0 10a6 6 0 0 1-6-6 6 6 0 0 1 6-6 6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12 20 8.69Z" / > < / svg >
< / label >
< input class = "md-option" data-md-color-media = "(prefers-color-scheme: dark)" data-md-color-scheme = "slate" data-md-color-primary = "indigo" data-md-color-accent = "indigo" aria-label = "Switch to light mode" type = "radio" name = "__palette" id = "__palette_2" >
< label class = "md-header__button md-icon" title = "Switch to light mode" for = "__palette_1" hidden >
< svg xmlns = "http://www.w3.org/2000/svg" viewBox = "0 0 24 24" > < path d = "M12 18c-.89 0-1.74-.2-2.5-.55C11.56 16.5 13 14.42 13 12c0-2.42-1.44-4.5-3.5-5.45C10.26 6.2 11.11 6 12 6a6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12 20 8.69Z" / > < / svg >
< / label >
< / form >
< label class = "md-header__button md-icon" for = "__search" >
< svg xmlns = "http://www.w3.org/2000/svg" viewBox = "0 0 24 24" > < path d = "M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z" / > < / svg >
< / label >
< div class = "md-search" data-md-component = "search" role = "dialog" >
< label class = "md-search__overlay" for = "__search" > < / label >
< div class = "md-search__inner" role = "search" >
< form class = "md-search__form" name = "search" >
< input type = "text" class = "md-search__input" name = "query" aria-label = "Search" placeholder = "Search" autocapitalize = "off" autocorrect = "off" autocomplete = "off" spellcheck = "false" data-md-component = "search-query" required >
< label class = "md-search__icon md-icon" for = "__search" >
< svg xmlns = "http://www.w3.org/2000/svg" viewBox = "0 0 24 24" > < path d = "M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.516 6.516 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5Z" / > < / svg >
< svg xmlns = "http://www.w3.org/2000/svg" viewBox = "0 0 24 24" > < path d = "M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12Z" / > < / svg >
< / label >
< nav class = "md-search__options" aria-label = "Search" >
< button type = "reset" class = "md-search__icon md-icon" title = "Clear" aria-label = "Clear" tabindex = "-1" >
< svg xmlns = "http://www.w3.org/2000/svg" viewBox = "0 0 24 24" > < path d = "M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12 19 6.41Z" / > < / svg >
< / button >
< / nav >
< / form >
< div class = "md-search__output" >
< div class = "md-search__scrollwrap" data-md-scrollfix >
< div class = "md-search-result" data-md-component = "search-result" >
< div class = "md-search-result__meta" >
Initializing search
< / div >
< ol class = "md-search-result__list" role = "presentation" > < / ol >
< / div >
< / div >
< / div >
< / div >
< / div >
2023-08-31 07:14:19 +00:00
< div class = "md-header__source" >
< a href = "https://github.com/txtdot/txtdot" title = "Go to repository" class = "md-source" data-md-component = "source" >
< div class = "md-source__icon md-icon" >
< svg xmlns = "http://www.w3.org/2000/svg" viewBox = "0 0 496 512" > <!-- ! Font Awesome Free 6.4.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc. --> < path d = "M165.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6zm-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3zm44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9zM244.8 8C106.1 8 0 113.3 0 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C428.2 457.8 496 362.9 496 252 496 113.3 383.5 8 244.8 8zM97.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1zm-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7zm32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1zm-11.4-14.7c-1.6 1-1.6 3.6 0 5.9 1.6 2.3 4.3 3.3 5.6 2.3 1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2z" / > < / svg >
< / div >
< div class = "md-source__repository" >
txtdot/txtdot
< / div >
< / a >
< / div >
2023-08-30 09:25:55 +00:00
< / nav >
< / header >
< div class = "md-container" data-md-component = "container" >
< main class = "md-main" data-md-component = "main" >
< div class = "md-main__inner md-grid" >
< div class = "md-sidebar md-sidebar--primary" data-md-component = "sidebar" data-md-type = "navigation" >
< div class = "md-sidebar__scrollwrap" >
< div class = "md-sidebar__inner" >
< nav class = "md-nav md-nav--primary" aria-label = "Navigation" data-md-level = "0" >
< label class = "md-nav__title" for = "__drawer" >
2024-05-27 09:38:37 +00:00
< a href = "." title = "txtdot" class = "md-nav__button md-logo" aria-label = "txtdot" data-md-component = "logo" >
2023-08-30 09:25:55 +00:00
< svg xmlns = "http://www.w3.org/2000/svg" viewBox = "0 0 24 24" > < path d = "M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54Z" / > < / svg >
< / a >
txtdot
< / label >
2023-08-31 07:14:19 +00:00
< div class = "md-nav__source" >
< a href = "https://github.com/txtdot/txtdot" title = "Go to repository" class = "md-source" data-md-component = "source" >
< div class = "md-source__icon md-icon" >
< svg xmlns = "http://www.w3.org/2000/svg" viewBox = "0 0 496 512" > <!-- ! Font Awesome Free 6.4.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc. --> < path d = "M165.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6zm-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3zm44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9zM244.8 8C106.1 8 0 113.3 0 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C428.2 457.8 496 362.9 496 252 496 113.3 383.5 8 244.8 8zM97.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1zm-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7zm32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1zm-11.4-14.7c-1.6 1-1.6 3.6 0 5.9 1.6 2.3 4.3 3.3 5.6 2.3 1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2z" / > < / svg >
< / div >
< div class = "md-source__repository" >
txtdot/txtdot
< / div >
< / a >
< / div >
2023-08-30 09:25:55 +00:00
< ul class = "md-nav__list" data-md-scrollfix >
2024-05-27 09:38:37 +00:00
2023-08-30 09:25:55 +00:00
2024-05-27 09:38:37 +00:00
< li class = "md-nav__item md-nav__item--active" >
< input class = "md-nav__toggle md-toggle" type = "checkbox" id = "__toc" >
< label class = "md-nav__link md-nav__link--active" for = "__toc" >
< span class = "md-ellipsis" >
Getting Started
< / span >
< span class = "md-nav__icon md-icon" > < / span >
< / label >
< a href = "." class = "md-nav__link md-nav__link--active" >
2023-08-30 09:25:55 +00:00
< span class = "md-ellipsis" >
Getting Started
< / span >
< / a >
2024-05-27 09:38:37 +00:00
< nav class = "md-nav md-nav--secondary" aria-label = "Table of contents" >
< label class = "md-nav__title" for = "__toc" >
< span class = "md-nav__icon md-icon" > < / span >
Table of contents
< / label >
< ul class = "md-nav__list" data-md-component = "toc" data-md-scrollfix >
< li class = "md-nav__item" >
< a href = "#what-is-this" class = "md-nav__link" >
What is this
< / a >
< / li >
< li class = "md-nav__item" >
< a href = "#how-to-use-it" class = "md-nav__link" >
How to use it
< / a >
< / li >
< li class = "md-nav__item" >
< a href = "#how-it-works" class = "md-nav__link" >
How it works
< / a >
< nav class = "md-nav" aria-label = "How it works" >
< ul class = "md-nav__list" >
< li class = "md-nav__item" >
< a href = "#plugins" class = "md-nav__link" >
Plugins
< / a >
< / li >
< / ul >
< / nav >
< / li >
< li class = "md-nav__item" >
< a href = "#engines" class = "md-nav__link" >
Engines
< / a >
< / li >
< li class = "md-nav__item" >
< a href = "#middlewares" class = "md-nav__link" >
Middlewares
< / a >
< / li >
< / ul >
< / nav >
2023-08-30 09:25:55 +00:00
< / li >
2023-08-31 07:14:19 +00:00
< li class = "md-nav__item" >
2024-05-27 09:46:55 +00:00
< a href = "selfhost/" class = "md-nav__link" >
2023-08-31 07:14:19 +00:00
< span class = "md-ellipsis" >
Self-Hosting
< / span >
< / a >
< / li >
2023-09-01 12:49:12 +00:00
< li class = "md-nav__item" >
2024-05-27 09:46:55 +00:00
< a href = "docker/" class = "md-nav__link" >
2023-09-01 12:49:12 +00:00
< span class = "md-ellipsis" >
Docker
< / span >
< / a >
< / li >
< li class = "md-nav__item" >
2024-05-27 09:46:55 +00:00
< a href = "env/" class = "md-nav__link" >
2023-09-01 12:49:12 +00:00
< span class = "md-ellipsis" >
Configuring
< / span >
< / a >
< / li >
< li class = "md-nav__item" >
2024-05-27 09:46:55 +00:00
< a href = "reverse/" class = "md-nav__link" >
2023-09-01 12:49:12 +00:00
< span class = "md-ellipsis" >
Reverse Proxy
< / span >
< / a >
< / li >
2023-08-30 09:25:55 +00:00
< / ul >
< / nav >
< / div >
< / div >
< / div >
< div class = "md-sidebar md-sidebar--secondary" data-md-component = "sidebar" data-md-type = "toc" >
< div class = "md-sidebar__scrollwrap" >
< div class = "md-sidebar__inner" >
< nav class = "md-nav md-nav--secondary" aria-label = "Table of contents" >
< label class = "md-nav__title" for = "__toc" >
< span class = "md-nav__icon md-icon" > < / span >
Table of contents
< / label >
< ul class = "md-nav__list" data-md-component = "toc" data-md-scrollfix >
< li class = "md-nav__item" >
< a href = "#what-is-this" class = "md-nav__link" >
What is this
< / a >
< / li >
< li class = "md-nav__item" >
< a href = "#how-to-use-it" class = "md-nav__link" >
How to use it
< / a >
< / li >
< li class = "md-nav__item" >
< a href = "#how-it-works" class = "md-nav__link" >
How it works
< / a >
2024-05-27 09:35:13 +00:00
< nav class = "md-nav" aria-label = "How it works" >
< ul class = "md-nav__list" >
< li class = "md-nav__item" >
< a href = "#plugins" class = "md-nav__link" >
Plugins
< / a >
< / li >
< / ul >
< / nav >
< / li >
< li class = "md-nav__item" >
< a href = "#engines" class = "md-nav__link" >
Engines
< / a >
< / li >
< li class = "md-nav__item" >
< a href = "#middlewares" class = "md-nav__link" >
Middlewares
< / a >
2023-08-30 09:25:55 +00:00
< / li >
< / ul >
< / nav >
< / div >
< / div >
< / div >
< div class = "md-content" data-md-component = "content" >
< article class = "md-content__inner md-typeset" >
2023-08-31 07:14:19 +00:00
2023-08-30 09:25:55 +00:00
< h1 id = "getting-started" > Getting Started< / h1 >
< h2 id = "what-is-this" > What is this< / h2 >
< p > < em > txtdot< / em > is a proxy that requests the page by the given URL,
extracts only useful data including text, links, pictures and tables,
and returns it as an HTML page with a minimalistic design
optimized for text reading.< / p >
< p > < em > txtdot< / em > increases the loading speed and reduces client's bandwidth usage
2023-08-31 07:14:19 +00:00
since no unnecessary code and no scripts are transferred.
2023-08-30 09:25:55 +00:00
Also, you won't see any advertisement (unless it's a static picture that is hard to detect as ads).
There are no trackers too.< / p >
< h2 id = "how-to-use-it" > How to use it< / h2 >
< p > < em > txtdot< / em > is an open source software, so everyone can host it on his own server.
The official instance is < a href = "https://txt.dc09.ru" > txt.dc09.ru< / a > ,
the list of all instances is < a href = "https://github.com/txtdot/instances" > here< / a > .< / p >
< p > On the main page, there's a handy form where you can
specify a URL, choose an engine and a format for parsed data.
On the < code > /get< / code > page, "Home" button returns you to < code > /< / code > ,
"Original page" opens the entered URL in the same window without txtdot proxy.< / p >
< p > The latest docs for API endpoints can be found < a href = "https://txt.dc09.ru/doc" > here< / a > .
For handy JSON API, use < code > /api/parse< / code > returning an engine result object (see below).
For pure HTML response, use < code > /api/raw-html< / code > .
Note that both API and browser endpoints on txt.dc09.ru
are ratelimited to 2 requests per second.< / p >
< h2 id = "how-it-works" > How it works< / h2 >
< p > This project exists thanks to great Mozilla's
< a href = "https://github.com/mozilla/readability" > Readability.js< / a > library.
The initial idea was to process HTML with it on the server
so the client does not need to download and execute heavy JS,
doesn't need to use an adblock.< / p >
2024-05-27 09:35:13 +00:00
< p > Readability performs its work very well in most cases.< / p >
2023-08-30 09:25:55 +00:00
< p > If an < code > ?engine=< / code > parameter wasn't passed, but txtdot found
that a specific engine is assigned to the requested domain,
2024-05-27 09:35:13 +00:00
for example, < code > "stackoverflow.com": engines.StackOverflow< / code > ,
2023-08-30 09:25:55 +00:00
it uses that engine to process the URL.
Otherwise, the page is parsed with the engine assigned to < code > *< / code > (it's Readability).< / p >
2024-05-27 09:35:13 +00:00
< h3 id = "plugins" > Plugins< / h3 >
< p > Readability is good, but now always, so < a href = "https://github.com/artegoser" > artegoser< / a > wrote the basis of the code
keeping in mind that we'll extend txtdot with other < em > engines< / em > .
Back then, it was functions taking a URL as a parameter,
returning an object that contains extracted HTML and plain text, page title and language.
The object is rendered with ejs template (or, in < code > /api/parse< / code > , just sent as JSON).< / p >
< p > But after a while it became unwieldy and we decided to create a monorepo. We created classes Engines, Middlewares that handle the necessary parts. Now you can create such functions for different domains, and routes. Also we added support for JSX for simplifying the code of plugins.< / p >
< h2 id = "engines" > Engines< / h2 >
< p > Creation of engines is easy.< / p >
< pre > < code class = "language-ts" > import { Engine, Route } from " @txtdot/sdk" ;
const Readability = new Engine(
" Readability" , // Name of the engine
" Engine for parsing content with Readability" , // Description
[" *" ] // Domains that use this engine
);
Readability.route(" *path" , async (input, ro: Route< { path: string }> ) => {
// ...
// If any of the parameters except content is empty, txtdot will try to extract it from the page automatically
return {
content: parsed.content,
title: parsed.title,
lang: parsed.lang,
};
});
< / code > < / pre >
< h2 id = "middlewares" > Middlewares< / h2 >
< p > Creation of middlewares similar to engines.< / p >
< pre > < code class = "language-tsx" > import { Middleware } from " @txtdot/sdk" ;
const Highlight = new Middleware(
" Highlight" ,
" Highlights code with highlight.js only when needed" ,
[" *" ]
);
Highlight.use(async (input, ro, out) => {
if (out.content.indexOf(" < code" ) !== -1)
return {
...out,
content: < Highlighter content={out.content} /> ,
};
return out;
});
< / code > < / pre >
2023-08-30 09:25:55 +00:00
< / article >
< / div >
< / div >
< / main >
< footer class = "md-footer" >
< div class = "md-footer-meta md-typeset" >
< div class = "md-footer-meta__inner md-grid" >
< div class = "md-copyright" >
Made with
< a href = "https://squidfunk.github.io/mkdocs-material/" target = "_blank" rel = "noopener" >
Material for MkDocs
< / a >
< / div >
< / div >
< / div >
< / footer >
< / div >
< div class = "md-dialog" data-md-component = "dialog" >
< div class = "md-dialog__inner md-typeset" > < / div >
< / div >
2024-05-27 09:38:37 +00:00
< script id = "__config" type = "application/json" > { "base" : "." , "features" : [ ] , "search" : "assets/javascripts/workers/search.dfff1995.min.js" , "translations" : { "clipboard.copied" : "Copied to clipboard" , "clipboard.copy" : "Copy to clipboard" , "search.result.more.one" : "1 more on this page" , "search.result.more.other" : "# more on this page" , "search.result.none" : "No matching documents" , "search.result.one" : "1 matching document" , "search.result.other" : "# matching documents" , "search.result.placeholder" : "Type to start searching" , "search.result.term.missing" : "Missing" , "select.version" : "Select version" } } < / script >
2023-08-30 09:25:55 +00:00
2024-05-27 09:38:37 +00:00
< script src = "assets/javascripts/bundle.78eede0e.min.js" > < / script >
2023-08-30 09:25:55 +00:00
< / body >
< / html >
