Essentially, I’d like to have pictrs delete all of the images that aren’t uploaded by my users, because my storage usage was going through the roof, so I just disabled the proxying of images. Here is my config:

x-logging: &default-logging
  driver: "json-file"
  options:
    max-size: "50m"
    max-file: "4"

services:
  proxy:
    image: docker.io/library/nginx
    volumes:
      - ./nginx_internal.conf:/etc/nginx/nginx.conf:ro,Z
      - ./proxy_params:/etc/nginx/proxy_params:ro,Z
    restart: always
    logging: *default-logging
    depends_on:
      - pictrs
      - lemmy-ui
    labels:
      - traefik.enable=true
      - traefik.http.routers.http-lemmy.entryPoints=http
      - traefik.http.routers.http-lemmy.rule=Host(`gregtech.eu`)
      - traefik.http.middlewares.https_redirect.redirectscheme.scheme=https
      - traefik.http.middlewares.https_redirect.redirectscheme.permanent=true
      - traefik.http.routers.http-lemmy.middlewares=https_redirect
      - traefik.http.routers.https-lemmy.entryPoints=https
      - traefik.http.routers.https-lemmy.rule=Host(`gregtech.eu`)
      - traefik.http.routers.https-lemmy.service=lemmy
      - traefik.http.routers.https-lemmy.tls=true
      - traefik.http.services.lemmy.loadbalancer.server.port=8536
      - traefik.http.routers.https-lemmy.tls.certResolver=le-ssl


  lemmy:
    image: dessalines/lemmy:0.19.8
    hostname: lemmy
    restart: always
    logging: *default-logging
    volumes:
      - ./lemmy.hjson:/config/config.hjson:Z
    depends_on:
      - postgres
      - pictrs
    networks:
      - default
      - database

  lemmy-ui:
    image: ghcr.io/xyphyn/photon:latest
    restart: always
    logging: *default-logging
    environment:
      - PUBLIC_INSTANCE_URL=gregtech.eu
      - PUBLIC_MIGRATE_COOKIE=true
#      - PUBLIC_SSR_ENABLED=true
      - PUBLIC_DEFAULT_FEED=All
      - PUBLIC_DEFAULT_FEED_SORT=Hot
      - PUBLIC_DEFAULT_COMMENT_SORT=Top
      - PUBLIC_LOCK_TO_INSTANCE=false



  pictrs:
    image: docker.io/asonix/pictrs:0.5
    # this needs to match the pictrs url in lemmy.hjson
    hostname: pictrs
    # we can set options to pictrs like this, here we set max. image size and forced format for conversion
    # entrypoint: /sbin/tini -- /usr/local/bin/pict-rs -p /mnt -m 4 --image-format webp
    #entrypoint: /sbin/tini -- /usr/local/bin/pict-rs run  --max-file-count 10  --media-max-file-size 500 --media-retention-proxy 10d --media-retention-variants 10d  filesystem sled -p /mnt
    user: 991:991
    environment:
      - PICTRS__STORE__TYPE=object_storage
      - PICTRS__STORE__ENDPOINT=https://s3.eu-central-003.backblazeb2.com/
      - PICTRS__STORE__BUCKET_NAME=gregtech-lemmy
      - PICTRS__STORE__REGION=eu-central
      - PICTRS__STORE__USE_PATH_STYLE=false
      - PICTRS__STORE__ACCESS_KEY=redacted
      - PICTRS__STORE__SECRET_KEY=redacted
      - PICTRS__MEDIA__RETENTION__VARIANTS=0d
      - PICTRS__MEDIA__RETENTION__PROXY=0d
      - PICTRS__SERVER__API_KEY=redacted_api_key
      #- PICTRS__MEDIA__IMAGE__FORMAT=webp
      #- PICTRS__MEDIA__IMAGE__QUALITY__WEBP=50
      #- PICTRS__MEDIA__ANIMATION__QUALITY=50
    volumes:
      - ./volumes/pictrs:/mnt:Z
    restart: always
    logging: *default-logging

  postgres:
    image: docker.io/postgres:16-alpine
    hostname: postgres
    volumes:
      - ./volumes/postgres:/var/lib/postgresql/data:Z
      #- ./customPostgresql.conf:/etc/postgresql.conf:Z
    restart: always
    #command: postgres -c config_file=/etc/postgresql.conf
    shm_size: 256M
    logging: *default-logging
    environment:
      - POSTGRES_PASSWORD=password
      - POSTGRES_USER=lemmy
      - POSTGRES_DB=lemmy
    networks:
      - database
  postfix:
    image: docker.io/mwader/postfix-relay
    restart: "always"
    logging: *default-logging

networks:
  default:
    name: traefik_access
    external: true
  database:
  • stinky@redlemmy.com
    link
    fedilink
    English
    arrow-up
    2
    ·
    8 days ago

    i’m trying to learn more about hosting my own instance

    are you willing to share your setup? docker/self hosted? storage is something i understand so i’m starting with that

    • Gregor@gregtech.euOP
      link
      fedilink
      arrow-up
      4
      ·
      8 days ago

      Here is my setup (make sure to read the comments). If you have any additional questions please ask, I love helping people with sysadmin stuff. So, I use Traefik as a reverse proxy, here’s its docker compose: services:

      services:
      
        traefik:
          image: traefik:latest
          ports:
            - 80:80 
            - 443:443
            - 8080:8080 # for Matrix
          environment:
            - CF_DNS_API_TOKEN=redacted # for my Cloudflare setup, you probably wouldn't need Cloudflare
          volumes:
            - /var/run/docker.sock:/var/run/docker.sock
            - ./traefik.yml:/etc/traefik/traefik.yml
            - ./acme.json:/acme.json
            - ./routes/:/routes
      networks:
        default:
          name: traefik_access # I have this so that I can simply put a networks block at the bottom of other docker compose files and traefik can then access it and proxy the traffic, if I put a labels block in the services I want to proxy, I'll also provide that.
      
      

      Next, we have my configuration file for Traefik, traefik.yml:

      certificatesResolvers:
        le-ssl: #for ssl certs
          acme:
            email: gregor@gregtech.eu
            storage: /acme.json
            dnsChallenge:
              provider: cloudflare #for cloudflare, you might want to change this somehow if you aren't going to use cloudflare, you can ask me if you need any help with this
              resolvers:
                - "1.1.1.1:53"
                - "1.0.0.1:53"
      
      providers:
        docker:
          exposedByDefault: false
          network: traefik_access
      
      log:
        level: DEBUG
      accessLog: {}
      
      api:
        dashboard: false
        insecure: false
      
      entryPoints:
        http:
          address: ":80"
        https:
          address: ":443"
      

      Not much to say about that one, following up we have my lemmy docker compose file, which I dare say is highly chaotic (I will still write the code comments tho):

      x-logging: &default-logging
        driver: "json-file"
        options:
          max-size: "50m"
          max-file: "4"
      
      services:
        proxy:
          image: docker.io/library/nginx
          volumes:
            - ./nginx_internal.conf:/etc/nginx/nginx.conf:ro,Z
            - ./proxy_params:/etc/nginx/proxy_params:ro,Z
          restart: always
          logging: *default-logging
          depends_on:
            - pictrs
            - lemmy-ui
          labels: #the ugly part: proxying. Here traefik proxies the traffic to nginx, of which the config I will provide.
            - traefik.enable=true
            - traefik.http.routers.http-lemmy.entryPoints=http
            - traefik.http.routers.http-lemmy.rule=Host(`gregtech.eu`)
            - traefik.http.middlewares.https_redirect.redirectscheme.scheme=https
            - traefik.http.middlewares.https_redirect.redirectscheme.permanent=true
            - traefik.http.routers.http-lemmy.middlewares=https_redirect
            - traefik.http.routers.https-lemmy.entryPoints=https
            - traefik.http.routers.https-lemmy.rule=Host(`gregtech.eu`)
            - traefik.http.routers.https-lemmy.service=lemmy
            - traefik.http.routers.https-lemmy.tls=true
            - traefik.http.services.lemmy.loadbalancer.server.port=8536
            - traefik.http.routers.https-lemmy.tls.certResolver=le-ssl
      
      
        lemmy:
          image: dessalines/lemmy:0.19.8
          hostname: lemmy
          restart: always
          logging: *default-logging
          volumes:
            - ./lemmy.hjson:/config/config.hjson:Z
          depends_on:
            - postgres
            - pictrs
          networks:
            - default
            - database
      
        lemmy-ui:
          image: ghcr.io/xyphyn/photon:latest # The lemmy UI accessible at gregtech.eu is actually not the official one, it's Photon
          restart: always
          logging: *default-logging
          environment:
            - PUBLIC_INSTANCE_URL=gregtech.eu
            - PUBLIC_MIGRATE_COOKIE=true
            - PUBLIC_DEFAULT_FEED=All
            - PUBLIC_DEFAULT_FEED_SORT=Hot
            - PUBLIC_DEFAULT_COMMENT_SORT=Top
            - PUBLIC_LOCK_TO_INSTANCE=false
      
      
      
        pictrs:
          image: docker.io/asonix/pictrs:0.5
          # this needs to match the pictrs url in lemmy.hjson
          hostname: pictrs
          # we can set options to pictrs like this, here we set max. image size and forced format for conversion
          # entrypoint: /sbin/tini -- /usr/local/bin/pict-rs -p /mnt -m 4 --image-format webp
          #entrypoint: /sbin/tini -- /usr/local/bin/pict-rs run  --max-file-count 10  --media-max-file-size 500 --media-retention-proxy 10d --media-retention-variants 10d  filesystem sled -p /mnt
          user: 991:991
          environment:
            - PICTRS__STORE__TYPE=object_storage #well, now comes the storage. I use backblaze , though you can also simply store them on-disk, or on another S3 storage provider.
            - PICTRS__STORE__ENDPOINT=https://s3.eu-central-003.backblazeb2.com/
            - PICTRS__STORE__BUCKET_NAME=gregtech-lemmy
            - PICTRS__STORE__REGION=eu-central
            - PICTRS__STORE__USE_PATH_STYLE=false
            - PICTRS__STORE__ACCESS_KEY=redacted
            - PICTRS__STORE__SECRET_KEY=redacted
            - PICTRS__MEDIA__RETENTION__VARIANTS=0d
            - PICTRS__MEDIA__RETENTION__PROXY=0d
            - PICTRS__SERVER__API_KEY=redacted #needed if you want to delete images
      
          volumes:
            - ./volumes/pictrs:/mnt:Z
          restart: always
          logging: *default-logging
      
        postgres:
          image: docker.io/postgres:16-alpine
          hostname: postgres
          volumes:
            - ./volumes/postgres:/var/lib/postgresql/data:Z
          restart: always
          shm_size: 256M
          logging: *default-logging
          environment:
            - POSTGRES_PASSWORD=password
            - POSTGRES_USER=lemmy
            - POSTGRES_DB=lemmy #this is just the db password
          networks:
            - database
        postfix:
          image: docker.io/mwader/postfix-relay
          restart: "always"
          logging: *default-logging
      
      
      networks:
        default:
          name: traefik_access #allows traefik to access these services for proxying
          external: true
        database:
      

      And here’s the nginx config, (nginx_internal.conf) I kinda forgot how it works lmao I wrote it like half a year ago:

      worker_processes auto;
      
      events {
          worker_connections 1024;
      }
      
      http {
          # Docker internal DNS IP so we always get the newer containers without having to 
          # restart/reload the docker container / nginx configuration
          resolver 127.0.0.11 valid=5s;
      
          # set the real_ip when from docker internal ranges. Ensuring our internal nginx
          # container can always see the correct ips in the logs
          set_real_ip_from 10.0.0.0/8;
          set_real_ip_from 172.16.0.0/12;
          set_real_ip_from 192.168.0.0/16;
      
          # We construct a string consistent of the "request method" and "http accept header"
          # and then apply soem ~simply regexp matches to that combination to decide on the
          # HTTP upstream we should proxy the request to.
          #
          # Example strings:
          #
          #   "GET:application/activity+json"
          #   "GET:text/html"
          #   "POST:application/activity+json"
          #
          # You can see some basic match tests in this regex101 matching this configuration
          # https://regex101.com/r/vwMJNc/1
          #
          # Learn more about nginx maps here http://nginx.org/en/docs/http/ngx_http_map_module.html
          map "$request_method:$http_accept" $proxpass {
              # If no explicit matches exists below, send traffic to lemmy-ui
              default "http://lemmy-ui:3000/";
      
              # GET/HEAD requests that accepts ActivityPub or Linked Data JSON should go to lemmy.
              #
              # These requests are used by Mastodon and other fediverse instances to look up profile information,
              # discover site information and so on.
              "~^(?:GET|HEAD):.*?application\/(?:activity|ld)\+json" "http://lemmy:8536/";
      
              # All non-GET/HEAD requests should go to lemmy
              #
              # Rather than calling out POST, PUT, DELETE, PATCH, CONNECT and all the verbs manually
              # we simply negate the GET|HEAD pattern from above and accept all possibly $http_accept values
              "~^(?!(GET|HEAD)).*:" "http://lemmy:8536/";
          }
      
          server {
              set $lemmy_ui "lemmy-ui:3000"; #these are the internal ports for the services
              set $lemmy "lemmy:8536";
              # this is the port inside docker, not the public one yet
              listen 1236;
              listen 8536;
      
              # change if needed, this is facing the public web
              server_name localhost;
              server_tokens off;
      
              # Upload limit, relevant for pictrs
              client_max_body_size 20M;
      
              # Send actual client IP upstream
              include proxy_params;
      
              # frontend general requests
              location / {
                  proxy_pass $proxpass;
                  rewrite ^(.+)/+$ $1 permanent;
              }
      
              # security.txt
              location = /.well-known/security.txt {
                  proxy_pass "http://$lemmy_ui";
              }
      
              # backend
              location ~ ^/(api|pictrs|feeds|nodeinfo|.well-known|version|sitemap.xml) {
                  proxy_pass "http://$lemmy";
      
                  # Send actual client IP upstream
                  include proxy_params;
              }
          }
      }
      
      

      And finally, we have lemmy’s config, lemmy.hjson:

      {
        # for more info about the config, check out the documentation
        # https://join-lemmy.org/docs/en/administration/configuration.html
      #make sure to check this out
        database: {
          host: postgres
          password: "password"
          # Alternative way:
          #uri: "postgresql://lemmy:{{ postgres_password }}@postgres/lemmy"
        }
        hostname: "gregtech.eu"
        pictrs: {
          url: "http://pictrs:8080/"
          api_key: "redacted"
          image_mode: "None" #to not proxy images, can be changed if you'd like to have the images proxied by your server
      
        }
        email: { #for sending email
          smtp_server: "smtp.sendgrid.net:587"
          smtp_from_address: "lemmy@lemmy.gregtech.eu"
          tls_type: "starttls"
          smtp_login: "redacted"
          smtp_password: "redacted"
        }
         # Parameters for automatic configuration of new instance (only used at first start)
        setup: {
          # Username for the admin user
          admin_username: "gregor"
          # Password for the admin user. It must be between 10 and 60 characters.
          admin_password: "redacted"
          # Name of the site, can be changed later. Maximum 20 characters.
          site_name: "Gremmy"
          # Email for the admin user (optional, can be omitted and set later through the website)
          admin_email: "redacted"
        }
      }
      
      
      • stinky@redlemmy.com
        link
        fedilink
        English
        arrow-up
        3
        ·
        8 days ago

        this is incredible, thanks <3

        federation interests me but I’m a lazy noob. I’d like to “find instances that I’m not federated with” or “migrate all of my Subscriptions to another account” or “find communities that have been created since I last checked”

        I host with Elestio and I think that development of the underlying tech may not be easy while my setup is tightly managed by them

        but I will save your post and come back here with questions