Merge pull request #12 from dalf/remove_filtron_morty

Replace morty & filtron by redis & the limiter plugin

.env

@@ -5,10 +5,3 @@
 
 # SEARXNG_HOSTNAME=<host>
 # LETSENCRYPT_EMAIL=<email>
-
-# automatically update settings to the new version
-# comment this line if you made / will make some modifications to the settings
-SEARXNG_COMMAND=-f
-
-# use openssl rand -base64 33
-MORTY_KEY=ReplaceWithARealKey!

.gitignore

@@ -3,4 +3,4 @@
 searxng-docker.service
 caddy
 srv
-searx
+searxng/uwsgi.ini

Caddyfile

@@ -11,7 +11,9 @@
 
   @api {
         path /config
-	path /status
+        path /healthz
+        path /stats/errors
+        path /stats/checker
   }
 
   @static {
@@ -22,12 +24,12 @@
         not path /static/*
   }
 
-  @morty {
+  @imageproxy {
-        path /morty/*
+        path /image_proxy
   }
 
-  @notmorty {
+  @notimageproxy {
-        not path /morty/*
+        not path /image_proxy
   }
 
   header {
@@ -40,11 +42,8 @@
         # Prevent some browsers from MIME-sniffing a response away from the declared Content-Type
         X-Content-Type-Options "nosniff"
 
-        # Disallow the site to be rendered within a frame (clickjacking protection)
-        X-Frame-Options "SAMEORIGIN"
-
         # Disable some features
-        Permissions-Policy "accelerometer=();ambient-light-sensor=(); autoplay=();camera=();encrypted-media=();focus-without-user-activation=(); geolocation=();gyroscope=();magnetometer=();microphone=();midi=();payment=();picture-in-picture=(); speaker=();sync-xhr=();usb=();vr=()"
+        Permissions-Policy "accelerometer=(),ambient-light-sensor=(),autoplay=(),camera=(),encrypted-media=(),focus-without-user-activation=(),geolocation=(),gyroscope=(),magnetometer=(),microphone=(),midi=(),payment=(),picture-in-picture=(),speaker=(),sync-xhr=(),usb=(),vr=()"
 
         # Disable some features (legacy)
         Feature-Policy "accelerometer 'none';ambient-light-sensor 'none'; autoplay 'none';camera 'none';encrypted-media 'none';focus-without-user-activation 'none'; geolocation 'none';gyroscope 'none';magnetometer 'none';microphone 'none';midi 'none';payment 'none';picture-in-picture 'none'; speaker 'none';sync-xhr 'none';usb 'none';vr 'none'"
@@ -78,29 +77,21 @@
   }
 
   # CSP (see http://content-security-policy.com/ )
-  header @morty {
+  header @imageproxy {
-	Content-Security-Policy "default-src 'none'; style-src 'self' 'unsafe-inline'; form-action 'self'; frame-ancestors 'self'; base-uri 'self'; img-src 'self' data:; font-src 'self'; frame-src 'self'"
+        Content-Security-Policy "default-src 'none'; img-src 'self' data:"
   }
 
-  header @notmorty {
+  header @notimageproxy {
         Content-Security-Policy "upgrade-insecure-requests; default-src 'none'; script-src 'self'; style-src 'self' 'unsafe-inline'; form-action 'self'; font-src 'self'; frame-ancestors 'self'; base-uri 'self'; connect-src 'self' https://overpass-api.de; img-src 'self' data: https://*.tile.openstreetmap.org; frame-src https://www.youtube-nocookie.com https://player.vimeo.com https://www.dailymotion.com https://www.deezer.com https://www.mixcloud.com https://w.soundcloud.com https://embed.spotify.com"
   }
 
-  # Morty
+  # SearXNG
-  handle @morty {
-        reverse_proxy localhost:3000
-  }
-
-  # Filtron
   handle {
         encode zstd gzip
 
-        reverse_proxy localhost:4040 {
+        reverse_proxy localhost:8080 {
                header_up X-Forwarded-Port {http.request.port}
                header_up X-Forwarded-Proto {http.request.scheme}
-               header_up X-Forwarded-TlsProto {tls_protocol}
-               header_up X-Forwarded-TlsCipher {tls_cipher}
-               header_up X-Forwarded-HttpsProto {proto}
         }
   }
 

README.md

@@ -7,22 +7,22 @@ Create a new SearXNG  instance in five minutes using Docker
 | Name | Description | Docker image | Dockerfile |
 | -- | -- | -- | -- |
 | [Caddy](https://github.com/caddyserver/caddy) | Reverse proxy (create a LetsEncrypt certificate automatically) | [caddy/caddy:2-alpine](https://hub.docker.com/_/caddy) | [Dockerfile](https://github.com/caddyserver/caddy-docker) |
-| [Filtron](https://github.com/dalf/filtron) |  Filtering reverse HTTP proxy, bot and abuse protection | [dalf/filtron:latest](https://hub.docker.com/r/dalf/filtron) | See [asciimoo/filtron#4](https://github.com/asciimoo/filtron/pull/4) |
 | [SearXNG](https://github.com/searxng/searxng) | SearXNG by itself | [searxng/searxng:latest](https://hub.docker.com/r/searxng/searxng) | [Dockerfile](https://github.com/searxng/searxng/blob/master/Dockerfile) |
-| [Morty](https://github.com/dalf/morty) | Privacy aware web content sanitizer proxy as a service. | [dalf/morty:latest](https://hub.docker.com/r/dalf/morty) | [Dockerfile](https://github.com/dalf/morty/blob/master/Dockerfile) |
+| [Redis](https://github.com/redis/redis) | In-memory database | [redis:alpine](https://hub.docker.com/_/redis) | [Dockerfile-alpine.template](https://github.com/docker-library/redis/blob/master/Dockerfile-alpine.template) |
 
 ## How to use it
 - [Install docker](https://docs.docker.com/install/)
 - [Install docker-compose](https://docs.docker.com/compose/install/) (be sure that docker-compose version is at least 1.9.0)
 - only on MacOSX: ```brew install coreutils``` to install ```greadlink```
-- Get searxng-docker:
+- Get searxng-docker
   ```sh
   cd /usr/local
   git clone https://github.com/searxng/searxng-docker.git
   cd searxng-docker
   ```
-- Generate MORTY_KEY ```sed -i "s|ReplaceWithARealKey\!|$(openssl rand -base64 33)|g" .env```
+- Edit the [.env](https://github.com/searxng/searxng-docker/blob/master/.env) file to set the hostname and an email
-- Edit the other settings in [.env](https://github.com/searxng/searxng-docker/blob/master/.env) file according to your need
+- Generate the secret key ```sed -i "s|ultrasecretkey|$(openssl rand -hex 32)|g" searxng/settings.yml```
+- Edit the [searxng/settings.yml](https://github.com/searxng/searxng-docker/blob/master/searxng/settings.yml) file according to your need
 - Check everything is working: ```./start.sh```
 - ```cp searxng-docker.service.template searxng-docker.service```
 - edit the content of ```WorkingDirectory``` in the ```searxng-docker.service``` file (only if the installation path is different from /usr/local/searxng-docker)
@@ -34,9 +34,9 @@ Create a new SearXNG  instance in five minutes using Docker
 
 ## Note on the image proxy feature
 
-The SearXNG image proxy is activated by default using [Morty](https://github.com/dalf/morty).
+The SearXNG image proxy is activated by default.
 
-The default [Content-Security-Policy](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy) allow the browser to access to {SEARXNG_HOSTNAME} and ```https://*.tile.openstreetmap.org;```.
+The default [Content-Security-Policy](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy) allow the browser to access to ```${SEARXNG_HOSTNAME}``` and ```https://*.tile.openstreetmap.org;```.
 
 If some users wants to disable the image proxy, you have to modify [./Caddyfile](https://github.com/searxng/searxng-docker/blob/master/Caddyfile). Replace the ```img-src 'self' data: https://*.tile.openstreetmap.org;``` by ```img-src * data:;```.
 
@@ -52,17 +52,11 @@ It is possible to use the [extend feature](https://docs.docker.com/compose/exten
 
 ## Multi Architecture Docker images
 
-For now only the amd64 platform is supported.
+Supported architecture:
+- amd64
+- arm64
+- arm/v7
 
 ## How to update ?
 
-Check the content of [```update.sh```](https://github.com/searxng/searxng-docker/blob/master/update.sh).
+Check the content of [```update.sh```](https://github.com/searxng/searxng-docker/blob/master/update.sh)
-
-## Access to the Filtron API
-
-The [Filtron API](https://github.com/dalf/filtron#api) is available on ```http://localhost:4041```.
-
-For example, to display the loaded rules:
-```
-curl http://localhost:4041/rules | jq
-```

docker-compose.yaml

@@ -5,14 +5,13 @@ services:
   caddy:
     container_name: caddy
     image: caddy:2-alpine
-    restart: on-failure
     network_mode: host
     volumes:
       - ./Caddyfile:/etc/caddy/Caddyfile:ro
       - caddy-data:/data:rw
       - caddy-config:/config:rw
     environment:
-      - SEARXNG_HOSTNAME=${SEARXNG_HOSTNAME:-localhost}
+      - SEARXNG_HOSTNAME=${SEARXNG_HOSTNAME:-http://localhost:80}
       - SEARXNG_TLS=${LETSENCRYPT_EMAIL:-internal}
     cap_drop:
       - ALL
@@ -20,21 +19,20 @@ services:
       - NET_BIND_SERVICE
       - DAC_OVERRIDE
 
-  filtron:
+  redis:
-    container_name: filtron
+    container_name: redis
-    image: dalf/filtron
+    image: "redis:alpine"
-    restart: always
+    command: redis-server --save "" --appendonly "no"
-    ports:
-      - "127.0.0.1:4040:4040"
-      - "127.0.0.1:4041:4041"
     networks:
       - searxng
-    command: -listen 0.0.0.0:4040 -api 0.0.0.0:4041 -target searxng:8080
+    tmpfs:
-    volumes:
+      - /var/lib/redis
-      - ./rules.json:/etc/filtron/rules.json:rw
-    read_only: true
     cap_drop:
       - ALL
+    cap_add:
+      - SETGID
+      - SETUID
+      - DAC_OVERRIDE
 
   searxng:
     container_name: searxng
@@ -42,14 +40,12 @@ services:
     restart: always
     networks:
       - searxng
-    command: ${SEARXNG_COMMAND:-}
+    ports:
+     - "127.0.0.1:8080:8080"
     volumes:
       - ./searxng:/etc/searxng:rw
     environment:
-      - BIND_ADDRESS=0.0.0.0:8080
+      - SEARXNG_BASE_URL=https://${SEARXNG_HOSTNAME:-localhost}/
-      - BASE_URL=https://${SEARXNG_HOSTNAME:-localhost}/
-      - MORTY_URL=https://${SEARXNG_HOSTNAME:-localhost}/morty/
-      - MORTY_KEY=${MORTY_KEY}
     cap_drop:
       - ALL
     cap_add:
@@ -57,25 +53,11 @@ services:
       - SETGID
       - SETUID
       - DAC_OVERRIDE
-
-  morty:
-    container_name: morty
-    image: dalf/morty
-    restart: always
-    ports:
-      - "127.0.0.1:3000:3000"
-    networks:
-      - searxng
-    command: -timeout 6 -ipv6
-    environment:
-      - MORTY_KEY=${MORTY_KEY}
-      - MORTY_ADDRESS=0.0.0.0:3000
     logging:
-      driver: none
+      driver: "json-file"
-    read_only: true
+      options:
-    cap_drop:
+        max-size: "1m"
-      - ALL
+        max-file: "1"
-
 networks:
   searxng:
     ipam:

rules.json

@@ -1,147 +0,0 @@
-[
-    {
-        "name": "searx.space",
-        "filters": ["Header:X-Forwarded-For=nslookup(check.searx.space)"],
-        "stop": true,
-        "actions": [{ "name": "log"}]
-    },
-    {
-        "name": "IP limit, all paths",
-        "interval": 3,
-        "limit": 25,
-        "aggregations": ["Header:X-Forwarded-For"],
-        "actions": [
-            {"name": "block",
-             "params": {"message": "Rate limit exceeded, try again later."}}
-        ]
-    },
-    {
-        "name": "useragent limit, all paths",
-        "interval": 30,
-        "limit": 200,
-        "aggregations": ["Header:X-Forwarded-For", "Header:User-Agent"],
-        "stop": true,
-        "actions": [
-            {"name": "block",
-             "params": {"message": "Rate limit exceeded, try again later."}}
-        ]
-    },
-    {
-        "name": "search request",
-        "filters": ["Param:q", "Path=^(/|/search)$"],
-        "subrules": [
-            {
-                "name": "allow Firefox Android (issue #48 and #60)",
-                "filters": [
-                    "Param:q=^1$",
-                    "Header:User-Agent=(^MozacFetch/[0-9]{2,3}.[0-9].[0-9]+$|^Mozilla/5.0 \\(Android [0-9]{1,2}(.[0-9]{1,2}.[0-9]{1,2})?; Mobile; rv:[0-9]{2,3}.[0-9]\\) Gecko/[0-9]{2,3}.[0-9] Firefox/[0-9]{2,3}.[0-9]$)"
-                ],
-                "stop": true,
-                "actions": [{"name": "log"}]
-            },
-            {
-                "name": "robot agent forbidden",
-                "limit": 0,
-                "stop": true,
-                "filters": ["Header:User-Agent=([Cc][Uu][Rr][Ll]|[wW]get|Scrapy|splash|JavaFX|FeedFetcher|python-requests|Go-http-client|Java|Jakarta|okhttp|HttpClient|Jersey|Python|libwww-perl|Ruby|SynHttpClient|UniversalFeedParser)"],
-                "actions": [
-                    {"name": "block",
-                     "params": {"message": "Rate limit exceeded"}}
-                ]
-            },
-            {
-                "name": "bot forbidden",
-                "limit": 0,
-                "stop": true,
-                "filters": ["Header:User-Agent=(Googlebot|GoogleImageProxy|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT|Sogou|Abonti|Pixray|Spinn3r|SemrushBot|Exabot|ZmEu|BLEXBot|bitlybot)"],
-                "actions": [
-                    {"name": "block",
-                     "params": {"message": "Rate limit exceeded"}}
-                ]
-            },
-            {
-                "name": "block missing accept-language",
-                "filters": ["!Header:Accept-Language"],
-                "limit": 0,
-                "stop": true,
-                "actions": [
-                    {"name": "block",
-                     "params": {"message": "Rate limit exceeded"}}
-                ]
-            },
-            {
-                "name": "block Connection:close",
-                "filters": ["Header:Connection=close"],
-                "limit": 0,
-                "stop": true,
-                "actions": [
-                    {"name": "block",
-                     "params": {"message": "Rate limit exceeded"}}
-                ]
-            },
-            {
-                "name": "block no gzip support",
-                "filters": ["!Header:Accept-Encoding=(^gzip$|^gzip[;,]|[; ]gzip$|[; ]gzip[;,])"],
-                "limit": 0,
-                "stop": true,
-                "actions": [
-                    {"name": "block",
-                     "params": {"message": "Rate limit exceeded"}}
-                ]
-            },
-            {
-                "name": "block no deflate support",
-                "filters": ["!Header:Accept-Encoding=(^deflate$|^deflate[;,]|[; ]deflate$|[; ]deflate[;,])"],
-                "limit": 0,
-                "stop": true,
-                "actions": [
-                    {"name": "block",
-                     "params": {"message": "Rate limit exceeded"}}
-                ]
-            },
-            {
-                "name": "block accept everything",
-                "filters": ["!Header:Accept=text/html"],
-                "limit": 0,
-                "stop": true,
-                "actions": [
-                    {"name": "block",
-                     "params": {"message": "Rate limit exceeded"}}
-                ]
-            },
-            {
-                "name": "rss/json limit",
-                "interval": 3600,
-                "limit": 4,
-                "stop": true,
-                "filters": ["Param:format=(csv|json|rss)"],
-                "aggregations": ["Header:X-Forwarded-For"],
-                "actions": [
-                    {"name": "block",
-		     "params": {"message": "Rate limit exceeded, try again later."}}
-                ]
-            },
-            {
-                "name": "IP limit",
-                "interval": 3,
-                "limit": 3,
-                "aggregations": ["Header:X-Forwarded-For"],
-                "actions": [
-                    {"name": "block",
-                     "params": {"message": "Rate limit exceeded, try again later."}}
-                ]
-            },
-            {
-                "name": "IP and useragent limit",
-                "interval": 600,
-                "limit": 60,
-                "stop": true,
-                "aggregations": ["Header:X-Forwarded-For", "Header:User-Agent"],
-                "actions": [
-                    {"name": "block",
-                     "params": {"message": "Rate limit exceeded, try again later."}}
-		]
-            }
-        ]
-    }
-]

searxng/settings.yml

@@ -0,0 +1,11 @@
+# see https://docs.searxng.org/admin/engines/settings.html#use-default-settings
+use_default_settings: true
+server:
+  # base_url is defined in the SEARXNG_BASE_URL environment variable, see .env and docker-compose.yml
+  secret_key: "ultrasecretkey"  # change this!
+  limiter: true  # can be disabled for a private instance
+  image_proxy: true
+ui:
+  static_use_hash: true
+redis:
+  url: redis://redis:6379/0