From 1d4ebb513228c241421ca060f614a47e91b48f8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20J=C3=A4ger?= Date: Wed, 17 Jan 2024 17:15:56 +0100 Subject: [PATCH] minor changes and docs --- self-hosting/helm/README.md | 93 +++++++++++++++++++++++++++++------ self-hosting/helm/values.yaml | 16 +++--- 2 files changed, 88 insertions(+), 21 deletions(-) diff --git a/self-hosting/helm/README.md b/self-hosting/helm/README.md index 64ccf0f3c..4791a88ef 100644 --- a/self-hosting/helm/README.md +++ b/self-hosting/helm/README.md @@ -1,37 +1,100 @@ # Deployment using Helm -### Prerequisites +Helm chart to self-host Omnivore. -Make sure the Helm repository is installed as follows: +## Notes and General Information + +This helm chart uses docker images from [`sejaeger/omnivore-*`](https://hub.docker.com/u/sejaeger). If you want to use the Web-UI or build your own images, checkout `../.build-and-push-images.sh`. You will find some hard-coded environment variables (e.g., `PG_DB` or `PG_USER`), please don't change them! Those are also hard-coded in the code base and changing them will likely cause problems. Please have a look at [the values file](values.yaml) and [change it accordingly](https://github.com/bjw-s/helm-charts/blob/main/charts/library/common/values.yaml) to your setup, especially: postgres hostname, elasticsearch URL, omnivore URL. + +Omnivore requires Postgres (+vector extension!) and Elasticsearch to store its information. Please make sure to have them up and running. Using the bitnami Helm charts works perfectly fine. However, for Postgres you need to use a custom built image that contains the vector extension: [See this descriptions](https://github.com/pgvector/pgvector/issues/126#issuecomment-1589203644) for more information or simply use `sejaeger/postgres-vector` from docker hub. + +This setup uses a couple of secrets to safely store passwords, tokens and private information. It's your responsibility to generate them and create the following secretes accordingly. + +* omnivore-image-proxy-secret + * IMAGE_PROXY_SECRET +* omnivore-jwt-secret + * JWT_SECRET +* omnivore-sso-jwt-secret + * SSO_JWT_SECRET +* omnivore-pg-password + * PG_PASSWORD +* postgres-admin-user-and-password + * PGPASSWORD + * POSTGRES_USER +* elasticsearch-auth-secret + * ES_PASSWORD +* omnivore-content-fetch-verification-token + * VERIFICATION_TOKEN + + +## Deployment ```console helm repo add bjw-s https://bjw-s.github.io/helm-charts helm repo update ``` -### Deployment - In order to deploy the manifest for this example, issue the following command: ```console -helm install omnivore bjw-s/app-template --namespace omnivore --values values.yaml +helm install omnivore bjw-s/app-template --values values.yaml ``` This will apply the rendered manifest(s) to your cluster. -## Notes +## RSS Subscriptions -- using `sejaeger` docker images. You can build your own using: `build-and-push-images.sh` -- don't change the following because also hard coded: `PG_DB`, `PG_USER` -- requires postgres (+vector extension!) and elasticsearch -- make sure to change the values according to your setup, especially: postgres hostname, elasticsearch URL, omnivore URL -- information about possible setup see [https://github.com/bjw-s/helm-charts/blob/main/charts/library/common/values.yaml](https://github.com/bjw-s/helm-charts/blob/main/charts/library/common/values.yaml) +Currently, handling RSS subscriptions are not supported for self-hosted instances. However, you can use this simple tool for this: https://github.com/se-jaeger/omnivore-rss-handler-hack. + +Adding the following `controller` and `persistence` information triggers the rss-handler hourly. +```yaml +controllers: + rss-handler-hack: + type: cronjob + cronjob: + schedule: "*/60 * * * *" + failedJobsHistoryLimit: 1 + successfulJobsHistoryLimit: 3 + concurrencyPolicy: Forbid + containers: + rss-handler-hack: + image: + repository: sejaeger/omnivore-rss-handler-hack + tag: v0.2 + imagePullPolicy: IfNotPresent + env: + API_URL: "http://omnivore-omnivore-api:8080/api/graphql" + CACHE_FILE: "/home/cache.json" + FEEDS_FILE: "/home/feeds.json" + envFrom: + - secretRef: + name: omnivore-api-token + +persistence: + rss-handler-hack: + type: persistentVolumeClaim + existingClaim: omnivore-pvc + advancedMounts: + rss-handler-hack: # controller name + rss-handler-hack: # container name + - path: /home +``` + +`FEEDS_FILE` is used to define the subscriptions: +```json +{ + "blog": "https://blog.example/feed", + "another-blog": "https://another-blog.example/rss.xml", +} +``` + +`omnivore-api-token` secret contains a single key `API_TOKEN`, which can be generated using the Omnivore Web-UI. -# TODOs -- health checks -- RSS feeds -- Docs \ No newline at end of file +## Currently not Implemented + +* health checks +* resource limits \ No newline at end of file diff --git a/self-hosting/helm/values.yaml b/self-hosting/helm/values.yaml index 5cc2d0ff5..b7de95c73 100644 --- a/self-hosting/helm/values.yaml +++ b/self-hosting/helm/values.yaml @@ -9,7 +9,7 @@ controllers: api: image: repository: sejaeger/omnivore-api - tag: 54ccbd4 + tag: e44616b01 pullPolicy: IfNotPresent env: API_ENV: local @@ -18,10 +18,11 @@ controllers: PG_DB: omnivore PG_PORT: 5432 PG_POOL_MAX: 20 - CLIENT_URL: https://omnivore.example.com + CLIENT_URL: https://omnivore.example.com # URL any client can use to reach the server GATEWAY_URL: http://omnivore-api:8080/api ES_URL: http://elasticsearch.elasticsearch.svc.cluster.local:9200 ES_USERNAME: elastic + CONTENT_FETCH_URL: http://omnivore-content-fetch:8080/?token=$(VERIFICATION_TOKEN) envFrom: - secretRef: name: omnivore-image-proxy-secret @@ -34,12 +35,12 @@ controllers: - secretRef: name: elasticsearch-auth-secret - secretRef: - name: omnivore-content-fetch-token + name: omnivore-content-fetch-verification-token initContainers: migrate: image: repository: sejaeger/omnivore-migrate - tag: 54ccbd4 + tag: e44616b01 pullPolicy: IfNotPresent command: ["/bin/sh", "-c", "--"] args: ["./packages/db/setup.sh"] @@ -54,6 +55,9 @@ controllers: - secretRef: name: postgres-admin-user-and-password + # NOTE: You need to build this image for your specific setup because NEXT.js burns some environment variables into client code + # See: https://nextjs.org/docs/pages/building-your-application/configuring/environment-variables#bundling-environment-variables-for-the-browser + # Use the ../build-and-push-images.sh script for your convenience web: enabled: true type: deployment @@ -62,7 +66,7 @@ controllers: web: image: repository: sejaeger/omnivore-web - tag: 54ccbd4 + tag: e44616b01 pullPolicy: IfNotPresent content-fetch: @@ -73,7 +77,7 @@ controllers: content-fetch: image: repository: sejaeger/omnivore-content-fetch - tag: 54ccbd4 + tag: e44616b01 pullPolicy: IfNotPresent env: REST_BACKEND_ENDPOINT: http://omnivore-api:8080/api