Keyboard shortcuts

Press ← or → to navigate between chapters

Press S or / to search in the book

Press ? to show this help

Press Esc to hide this help

Nginx充当镜像源

想法

  • 缓存一些常用的软件源，但不想占用太大的本地存储空间
    全量同步下来的东西80%都是没用的

架构

graph LR;
    客户端 --> Nginx;
    Nginx --> 元数据;
    元数据 <--> |ByPass| 镜像站;
    软件包 <--> 镜像站;
    Nginx --> 软件包;
    软件包 <--> 硬盘缓存;

配置

Nginx主配置

# 开启代理缓存 cache
proxy_cache_path /data/nginx/cache levels=1:2 keys_zone=cache:35m inactive=365d max_size=64g min_free=64m;

Server配置

BYPASS 元数据 
# 这部分或许可以放到location里面，但需要配置很多location。
map $uri $cache_bypass {
    default                                                                     '';
    # ArchLinux / pacman
    '~^/(archlinux|archlinuxcn)/.*\.db(\.sig)?$'                                1;
    # Debian / Ubuntu / apt
    '~^/(debian-archive/)?(debian|debian-security)/.*/(In)?Release$'            1;
    '~^/(debian-archive/)?(debian|debian-security)/.*/Packages(\.(gz|bz2|xz))?' 1;
    '~^/docker-ce/linux/(debian|ubuntu|raspbian)/.*/(In)?Release$'              1;
    '~^/docker-ce/linux/(debian|ubuntu|raspbian)/.*/Packages(\.(gz|bz2|xz))?'   1;
    '~^/mysql/apt/.*/(In)?Release$'                                             1;
    '~^/mysql/apt/.*/Packages(\.(gz|bz2|xz))?'                                  1;
    '~^/proxmox/debian/.*/(In)?Release$'                                        1;
    '~^/proxmox/debian/.*/Packages(\.(gz|bz2|xz))?'                             1;
    '~^/zabbix/.*/(debian|ubuntu|raspbian)/.*/(In)?Release$'                    1;
    '~^/zabbix/.*/(debian|ubuntu|raspbian)/.*/Packages(\.(gz|bz2|xz))?'         1;
    # Fedora / CentOS / yum / dnf
    '~^/(centos|centos-vault|epel|opencloudos|tlinux)/.*/repodata/[^/]+$'       1;
    '~^/docker-ce/linux/(centos|fedora|rhel)/.*/repodata/[^/]+$'                1;
    '~^/mysql/yum/.*/repodata/[^/]+$'                                           1;
    '~^/zabbix/.*/(centos|fedora|rhel|amazonlinux)/.*/repodata/[^/]+$'          1;
    # Python / pip
    '~^/pypi/.*/$'                                                              1;
    '~^/pypi/.*/index.html?$'                                                   1;
    # Alpine / apk
    '~^/alpine/.*/APKINDEX\.?[^/]*$'                                            1;
}
镜像原站 
map $repo $backend {
    default         '';

    archlinux       aliyun;
    archlinuxcn     aliyun;
    debian-archive  aliyun;

    opencloudos     opencloudos;

    python          npmmirror;
    node            npmmirror;
    npm             npmmirror;
    electron        npmmirror;
    sqlite3         npmmirror;

    proxmox         ustc;

    alpine          tencent;
    centos          tencent;
    centos-vault    tencent;
    debian          tencent;
    debian-security tencent;
    docker-ce       tencent;
    elrepo          tencent;
    epel            tencent;
    epel-archive    tencent;
    mysql           tencent;
    pypi            tencent;
    tlinux          tencent;
    zabbix          tencent;
}
镜像原站URL前缀 
map $repo $prefix {
    default  '';
    python   '/binaries';
    node     '/binaries';
    npm      '/binaries';
    electron '/binaries';
    sqlite3  '/binaries';
}
原站主机名 
# 为什么这样配？因为 $upstream_last_server_name 只在商业版提供
# 如果用 proxy_pass 直接指向源站域名，就没办法复用TCP连接
map $backend $backend_host {
    aliyun      mirrors.aliyun.com;
    opencloudos mirrors.opencloudos.tech;
    ustc        mirrors.ustc.edu.cn;
    npmmirror   cdn.npmmirror.com;
    sustech     mirrors.sustech.edu.cn;
    tuna        mirrors.tuna.tsinghua.edu.cn;
    tencent     mirrors.tencent.com;
}
原站（支持keepalive） 
upstream aliyun {
    hash $remote_port; # 用客户端端口哈希，确保链接服用
    zone mirrors 10m;
    
    keepalive          32;
    keepalive_requests 256;
    
    resolver         127.0.0.11;
    resolver_timeout 2s;
    server  mirrors.aliyun.com:443 resolve;
}
upstream opencloudos {
    hash $remote_port; # 用客户端端口哈希，确保链接服用
    zone mirrors 10m;
    
    keepalive          32;
    keepalive_requests 256;
    
    resolver         127.0.0.11;
    resolver_timeout 2s;
    server  mirrors.opencloudos.tech:443 resolve;
}
upstream ustc {
    hash $remote_port; # 用客户端端口哈希，确保链接服用
    zone mirrors 10m;
    
    keepalive          32;
    keepalive_requests 256;
    
    resolver         127.0.0.11;
    resolver_timeout 2s;
    server  mirrors.ustc.edu.cn:443 resolve;
}
upstream npmmirror {
    hash $remote_port; # 用客户端端口哈希，确保链接服用
    zone mirrors 10m;
    
    keepalive          32;
    keepalive_requests 256;
    
    resolver         127.0.0.11;
    resolver_timeout 2s;
    server  cdn.npmmirror.com:443 resolve;
}
upstream sustech {
    hash $remote_port; # 用客户端端口哈希，确保链接服用
    zone mirrors 10m;
    
    keepalive          32;
    keepalive_requests 256;
    
    resolver         127.0.0.11;
    resolver_timeout 2s;
    server  mirrors.sustech.edu.cn:443 resolve;
}
upstream tuna {
    hash $remote_port; # 用客户端端口哈希，确保链接服用
    zone mirrors 10m;
    
    keepalive          32;
    keepalive_requests 256;
    
    resolver         127.0.0.11;
    resolver_timeout 2s;
    server  mirrors.tuna.tsinghua.edu.cn:443 resolve;
}
upstream tencent {
    hash $remote_port; # 用客户端端口哈希，确保链接服用
    zone mirrors 10m;
    
    keepalive          32;
    keepalive_requests 256;
    
    resolver         127.0.0.11;
    resolver_timeout 2s;
    server  mirrors.tencent.com:443 resolve;
}
镜像站 
server {
    listen [::]:80;
    listen [::]:443 ssl;
    listen [::]:443 quic;
    server_name mirror mirrors

    resolver         127.0.0.11;
    resolver_timeout 2s;

    # 如果是CentOS7，跳转到最新的Vault源
    location ^~ /centos/7 {
        rewrite '^/centos/7(?:\.\d\.\d{4})?(/.*)' /centos-vault/7.9.2009$1 last;
    }
    # 如果是CentOS7 Vault，但没指定版本，跳转到最新
    location ^~ /centos-vault/7/ {
        rewrite '^/centos-vault/7(?:\.\d\.\d{4})(/.*)' /centos-vault/7.9.2009$1 last;
    }
    # 如果是CentOS6，跳转到最新的Vault源
    location ^~ /centos/6 {
        rewrite '^/centos/6(?:\.\d+)?(/.*)' /centos-vault/6.10$1 last;
    }
    # 如果是CentOS6 Vault，但没指定版本，跳转到最新
    location ^~ /centos-vault/6/ {
        rewrite '^/centos-vault/6(?:\.\d+)(/.*)' /centos-vault/6.10$1 last;
    }
    # 如果是Debian但找不到包，跳转到Archive
    location @debian-failback {
        rewrite ^(/-/metadata)?(.*) /debian-archive$2 last;
    }
    # 如果是Centos但找不到包，跳转到Vault
    location @centos-failback {
        rewrite ^(/-/metadata)?/centos(/.*) /centos-vault$2 last;
    }
    # 如果是Epel但找不到包，跳转到Archive
    location @epel-failback {
        rewrite ^(/-/metadata)?/epel(/.*) /epel-archive$2 last;
    }
    # 主处理逻辑
    location ~* ^/(?<repo>[^/]+) {
        # 判断是否代理镜像
        if ($backend = '') {
            rewrite (.*) / last;
        }
        # 判断是否有前缀
        if ($prefix != '') {
            rewrite ^(.*)$ $prefix$1 break;
        }
        # 处理代理请求
        # 上游链接发送SNI
        proxy_ssl_name $backend_host;
        proxy_ssl_server_name on;
        # 复用SSL会话
        proxy_ssl_session_reuse on;
        # 验证上游证书
        proxy_ssl_verify              on;
        proxy_ssl_trusted_certificate /etc/ssl/cert.pem;
        proxy_ssl_certificate_cache   max=1000 inactive=1m valid=10m;
        # 上游链接加密算法
        proxy_ssl_ciphers TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256:TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256:TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256:TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256:TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384:TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384;
        # 上游链接协议版本
        proxy_ssl_protocols TLSv1.2 TLSv1.3;
        # 使用 HTTP 1.1 链接复用
        proxy_http_version     1.1;
        proxy_set_header       Host $backend_host;
        proxy_set_header       Connection "";
        proxy_pass             https://$backend;
        # 代理启用缓存
        proxy_cache            cache;
        proxy_cache_bypass     $cache_bypass;
        proxy_socket_keepalive on;
        # 分片下载（按网速配置）
        slice            16m;
        proxy_set_header Range $slice_range;
        
        proxy_buffering on;
        # 代理缓存配置
        # 不转换HEAD减少请求时间
        proxy_cache_convert_head off;
        proxy_cache_key          $request_method$scheme$backend_host$uri$slice_range;
        proxy_cache_min_uses     1;
        proxy_cache_revalidate   on;
        proxy_cache_use_stale    updating;
        proxy_cache_valid        200 206 31d;
        proxy_cache_valid        301 308 1h;
        proxy_cache_valid        302 307 304 5m;
        proxy_cache_valid        any     15s;
        
        proxy_hide_header Ali-Swift-Global-Savetime;
        proxy_hide_header EagleId;
        proxy_hide_header Timing-Allow-Origin;
        proxy_hide_header Via;
        proxy_hide_header X-Cache-Lookup;
        proxy_hide_header X-Cache;
        proxy_hide_header X-NWS-LOG-UUID;
        proxy_hide_header X-Swift-CacheTime;
        proxy_hide_header X-Swift-SaveTime;
        proxy_intercept_errors on;
        # Debian源和Debian Security源跳转
        if ($repo ~* ^(debian|debian-security)$) {
            error_page 404 = @debian-failback;
        }
        # CentOS源跳转
        if ($repo = centos) {
            error_page 404 = @centos-failback;
        }
        # Epel源跳转
        if ($repo = epel) {
            error_page 404 = @epel-failback;
        }
    }
    # 处理异常请求
    location / {
        add_header Content-Type text/plain always;
        return 404 '404 not found\n';
    }
}
当然也可以来点伪镜像站 
## 适用于docker构建加速，可不修改镜像源配置
server {
    listen      [::]:80;
    server_name deb.debian.org;

    resolver         127.0.0.11;
    resolver_timeout 2s;

    location ~* ^/(?<repo>debian|debian-security) {
        if ($backend = '') {
            rewrite (.*) / last;
        }

        # 上游链接发送SNI
        proxy_ssl_name $backend_host;
        proxy_ssl_server_name on;
        # 复用SSL会话
        proxy_ssl_session_reuse on;
        # 验证上游证书
        proxy_ssl_verify              on;
        proxy_ssl_trusted_certificate /etc/ssl/cert.pem;
        proxy_ssl_certificate_cache   max=1000 inactive=1m valid=10m;
        # 上游链接加密算法
        proxy_ssl_ciphers TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256:TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256:TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256:TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256:TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384:TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384;
        # 上游链接协议版本
        proxy_ssl_protocols TLSv1.2 TLSv1.3;
        # 使用 HTTP 1.1 链接复用
        proxy_http_version     1.1;
        proxy_set_header       Host $backend_host;
        proxy_set_header       Connection "";
        proxy_pass             https://$backend;
        # 代理启用缓存
        proxy_cache            cache;
        proxy_cache_bypass     $cache_bypass;
        proxy_socket_keepalive on;
        # 分片下载（按网速配置）
        slice            16m;
        proxy_set_header Range $slice_range;
        
        proxy_buffering on;
        # 代理缓存配置
        # 不转换HEAD减少请求时间
        proxy_cache_convert_head off;
        proxy_cache_key          $request_method$scheme$backend_host$uri$slice_range;
        proxy_cache_min_uses     1;
        proxy_cache_revalidate   on;
        proxy_cache_use_stale    updating;
        proxy_cache_valid        200 206 31d;
        proxy_cache_valid        301 308 1h;
        proxy_cache_valid        302 307 304 5m;
        proxy_cache_valid        any     15s;
        
        proxy_hide_header Ali-Swift-Global-Savetime;
        proxy_hide_header EagleId;
        proxy_hide_header Timing-Allow-Origin;
        proxy_hide_header Via;
        proxy_hide_header X-Cache-Lookup;
        proxy_hide_header X-Cache;
        proxy_hide_header X-NWS-LOG-UUID;
        proxy_hide_header X-Swift-CacheTime;
        proxy_hide_header X-Swift-SaveTime;
        proxy_intercept_errors on;
    }

    location / {
        add_header Content-Type text/plain always;
        return 404 '404 not found\n';
    }
}

server {
    listen      [::]:80;
    server_name archive.debian.org;

    resolver         127.0.0.11;
    resolver_timeout 2s;

    location ~* ^/(?<repo>debian-archive) {
        if ($backend = '') {
            rewrite (.*) / last;
        }

        # 上游链接发送SNI
        proxy_ssl_name $backend_host;
        proxy_ssl_server_name on;
        # 复用SSL会话
        proxy_ssl_session_reuse on;
        # 验证上游证书
        proxy_ssl_verify              on;
        proxy_ssl_trusted_certificate /etc/ssl/cert.pem;
        proxy_ssl_certificate_cache   max=1000 inactive=1m valid=10m;
        # 上游链接加密算法
        proxy_ssl_ciphers TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256:TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256:TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256:TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256:TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384:TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384;
        # 上游链接协议版本
        proxy_ssl_protocols TLSv1.2 TLSv1.3;
        # 使用 HTTP 1.1 链接复用
        proxy_http_version     1.1;
        proxy_set_header       Host $backend_host;
        proxy_set_header       Connection "";
        proxy_pass             https://$backend;
        # 代理启用缓存
        proxy_cache            cache;
        proxy_cache_bypass     $cache_bypass;
        proxy_socket_keepalive on;
        # 分片下载（按网速配置）
        slice            16m;
        proxy_set_header Range $slice_range;
        
        proxy_buffering on;
        # 代理缓存配置
        # 不转换HEAD减少请求时间
        proxy_cache_convert_head off;
        proxy_cache_key          $request_method$scheme$backend_host$uri$slice_range;
        proxy_cache_min_uses     1;
        proxy_cache_revalidate   on;
        proxy_cache_use_stale    updating;
        proxy_cache_valid        200 206 31d;
        proxy_cache_valid        301 308 1h;
        proxy_cache_valid        302 307 304 5m;
        proxy_cache_valid        any     15s;
        
        proxy_hide_header Ali-Swift-Global-Savetime;
        proxy_hide_header EagleId;
        proxy_hide_header Timing-Allow-Origin;
        proxy_hide_header Via;
        proxy_hide_header X-Cache-Lookup;
        proxy_hide_header X-Cache;
        proxy_hide_header X-NWS-LOG-UUID;
        proxy_hide_header X-Swift-CacheTime;
        proxy_hide_header X-Swift-SaveTime;
        proxy_intercept_errors on;
    }

    location ~* ^/(?<repo>debian|debian-security) {
        rewrite (.*) /debian-archive$1 last;
    }

    location / {
        add_header Content-Type text/plain always;
        return 404 '404 not found\n';
    }
}
Last change: , commit: d8e4d6b