terraform/tf_registry
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

init: registry server + operator from VM

Browse Source
This commit is contained in:
“Naeel”
2026-07-01 17:37:08 +04:00
commit dd1007d24c
67 changed files with 3293 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/.dockerignore
+47
View File
@@ -0,0 +1,47 @@
# Exclude files not needed in Docker build context for registry-server
.git
.gitignore
.dockerignore
# Docs / markdown
*.md
docs/
# Other services / infra
operator/
registrykeys/
scripts/
charts/
k8s/
# Secrets & keys
secrets/
*.pem
*.key
*.asc
*.gpg
*.pgp
# Artefacts / temp
artifacts/
*.har
HAR/
universal_rebuild/
tools/
devops/
# IDE / OS
.idea/
.vscode/
*.swp
*.swo
*~
.DS_Store
# Go test/coverage
*.test
*.out
coverage.out
# Logs
*.log
server/README.md
+56
View File
@@ -0,0 +1,56 @@
# Registry Server Build
This folder holds the standalone registry server build and helper scripts.
## Provider build and upload
Script: `build-provider.sh`
What it does:
- Builds the provider from `universal_rebuild` for 3 platforms: linux_amd64, windows_amd64, darwin_amd64
- Creates ZIPs and SHA256SUMS
- Signs SHA256SUMS with a local GPG key file
- Uploads artifacts to S3 for the registry server to serve
### Requirements
- Go 1.22+
- `python3` (for zipfile)
- `gpg`
- `mc` (MinIO/S3 client)
### GPG key file
Store the private key at:
- `secrets/private_key.asc` (ignored by git)
The public key is at:
- `secrets/public_key.asc`
The registry server embeds the public key it returns to Terraform. After regenerating keys:
- Update the embedded ASCII Armor in `registry-server-build/main.go` and `operator/cmd/registry/main.go`.
- Rebuild and redeploy the registry server.
- Re-upload provider artifacts signed with the new private key.
### Environment
Set these variables before running:
- `S3_ENDPOINT` (example: `s3.msk-1.ngcloud.ru`)
- `S3_ACCESS_KEY`
- `S3_SECRET_KEY`
Optional overrides:
- `REGISTRY_HOSTNAME` (default: `terra.k8c.ru`)
- `NAMESPACE` (default: `nubes`)
- `NAME` (default: `nubes`)
- `S3_BUCKET` (default: `terraform-registry`)
### Usage
```bash
export S3_ENDPOINT="s3.msk-1.ngcloud.ru"
export S3_ACCESS_KEY="..."
export S3_SECRET_KEY="..."
./registry-server-build/build-provider.sh 2.0.2
```
Artifacts are uploaded to:
`registry/<bucket>/<hostname>/<namespace>/<name>/<version>/`
server/build-provider.sh
Executable
+115
View File
@@ -0,0 +1,115 @@
#!/usr/bin/env bash
# Build and upload the Terraform provider to the registry S3 bucket.
#
# GPG signing key requirements:
# - A private key is required for signing SHA256SUMS.
# - The private key must be available at:
# ${ROOT_DIR}/secrets/private_key.asc
# - The private key is intentionally ignored by git.
#
# If the key is missing, generate it with a no-passphrase batch run, then
# export it to the expected location. Example (email only, no passphrase):
# GPG_DIR=${ROOT_DIR}/secrets
# GNUPGHOME=$(mktemp -d)
# cat > /tmp/gpg_batch <<'EOF'
# %no-protection
# Key-Type: RSA
# Key-Length: 3072
# Subkey-Type: RSA
# Subkey-Length: 3072
# Name-Real: tazet@narod.ru
# Name-Email: tazet@narod.ru
# Expire-Date: 0
# EOF
# gpg --batch --homedir "$GNUPGHOME" --gen-key /tmp/gpg_batch
# gpg --batch --homedir "$GNUPGHOME" --armor --export-secret-keys > "$GPG_DIR/private_key.asc"
# gpg --batch --homedir "$GNUPGHOME" --armor --export > "$GPG_DIR/public_key.asc"
# rm -rf "$GNUPGHOME" /tmp/gpg_batch
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ROOT_DIR="${ROOT_DIR:-$(cd "${SCRIPT_DIR}/.." && pwd)}"
PROVIDER_DIR="${PROVIDER_DIR:-${ROOT_DIR}/universal_rebuild}"
BUILD_DIR="${BUILD_DIR:-${ROOT_DIR}/artifacts/provider_build}"
GPG_KEY_FILE="${GPG_KEY_FILE:-${ROOT_DIR}/secrets/private_key.asc}"
VERSION="${1:-2.0.2}"
REGISTRY_HOSTNAME="${REGISTRY_HOSTNAME:-terra.k8c.ru}"
NAMESPACE="${NAMESPACE:-nubes}"
PROVIDER_NAME="${PROVIDER_NAME:-nubes}"
S3_BUCKET="${S3_BUCKET:-terraform-registry}"
S3_ENDPOINT="${S3_ENDPOINT:-}"
S3_ACCESS_KEY="${S3_ACCESS_KEY:-}"
S3_SECRET_KEY="${S3_SECRET_KEY:-}"
if [[ -z "$S3_ENDPOINT" || -z "$S3_ACCESS_KEY" || -z "$S3_SECRET_KEY" ]]; then
echo "Error: S3_ENDPOINT/S3_ACCESS_KEY/S3_SECRET_KEY must be set" >&2
echo "Hint: see ${ROOT_DIR}/secrets/.s3cfg_registry" >&2
exit 2
fi
if [[ ! -f "$GPG_KEY_FILE" ]]; then
echo "Error: GPG key file not found at $GPG_KEY_FILE" >&2
exit 2
fi
if [[ ! -d "$PROVIDER_DIR" ]]; then
echo "Error: provider source dir not found: $PROVIDER_DIR" >&2
exit 2
fi
s3_endpoint_url="$S3_ENDPOINT"
if [[ "$s3_endpoint_url" != http* ]]; then
s3_endpoint_url="https://${s3_endpoint_url}"
fi
rm -rf "$BUILD_DIR"
mkdir -p "$BUILD_DIR"
build_and_zip() {
local os="$1"
local arch="$2"
local binary="terraform-provider-nubes_v${VERSION}"
if [[ "$os" == "windows" ]]; then
binary+=".exe"
fi
echo "Building for ${os}/${arch}..."
(cd "$PROVIDER_DIR" && CGO_ENABLED=0 GOOS="$os" GOARCH="$arch" go build -o "${BUILD_DIR}/${binary}" .)
(cd "$BUILD_DIR" && python3 -m zipfile -c "terraform-provider-nubes_${VERSION}_${os}_${arch}.zip" "$binary")
rm -f "${BUILD_DIR:?}/${binary}"
}
build_and_zip linux amd64
build_and_zip windows amd64
build_and_zip darwin amd64
( cd "$BUILD_DIR" && sha256sum *.zip > "terraform-provider-nubes_${VERSION}_SHA256SUMS" )
GNUPGHOME="$(mktemp -d)"
export GNUPGHOME
trap 'rm -rf "$GNUPGHOME"' EXIT
gpg --batch --import "$GPG_KEY_FILE" >/dev/null
key_id=$(gpg --list-keys --with-colons | awk -F: '/^pub/{print $5; exit}')
if [[ -z "$key_id" ]]; then
echo "Error: failed to read key id from $GPG_KEY_FILE" >&2
exit 2
fi
gpg --batch --yes --detach-sign --default-key "$key_id" \
--output "${BUILD_DIR}/terraform-provider-nubes_${VERSION}_SHA256SUMS.sig" \
"${BUILD_DIR}/terraform-provider-nubes_${VERSION}_SHA256SUMS"
mc alias set registry "$s3_endpoint_url" "$S3_ACCESS_KEY" "$S3_SECRET_KEY" --api S3v4
S3_PATH="registry/${S3_BUCKET}/${REGISTRY_HOSTNAME}/${NAMESPACE}/${PROVIDER_NAME}/${VERSION}/"
echo "Uploading to: ${S3_PATH}"
mc cp "${BUILD_DIR}"/*.zip "$S3_PATH"
mc cp "${BUILD_DIR}/terraform-provider-nubes_${VERSION}_SHA256SUMS" "$S3_PATH"
mc cp "${BUILD_DIR}/terraform-provider-nubes_${VERSION}_SHA256SUMS.sig" "$S3_PATH"
echo "Done. Version ${VERSION} uploaded."
server/docs.go
+116
View File
@@ -0,0 +1,116 @@
package main
import (
"context"
"fmt"
"io"
"log"
"mime"
"net/http"
"path/filepath"
"strings"
s3 "github.com/minio/minio-go/v7"
)
// parseDocsRequestPath parses paths like:
// /docs/<namespace>/<name>/<version>/... (rest may be empty)
func parseDocsRequestPath(p string) (namespace, name, version, rest string, err error) {
p = strings.TrimPrefix(p, "/")
p = strings.TrimPrefix(p, "docs/")
parts := strings.SplitN(p, "/", 4)
if len(parts) < 3 {
err = fmt.Errorf("invalid docs path: %s", p)
return
}
namespace = parts[0]
name = parts[1]
version = parts[2]
if len(parts) == 3 {
rest = ""
} else {
rest = parts[3]
}
return
}
// docsObjectKey builds the S3 key for a docs object given parsed parts.
func docsObjectKey(namespace, name, version, pathPart string) string {
clean := strings.TrimPrefix(pathPart, "/")
if clean == "" {
return fmt.Sprintf("docs/%s/%s/%s/index.html", namespace, name, version)
}
return fmt.Sprintf("docs/%s/%s/%s/%s", namespace, name, version, clean)
}
// tryCandidateKeys returns a list of keys to attempt for a given request path.
// Order matters: exact path first, then <path>/index.html, then top-level index.
func tryCandidateKeys(namespace, name, version, rest string) []string {
keys := []string{}
if rest == "" {
keys = append(keys, docsObjectKey(namespace, name, version, "index.html"))
return keys
}
// exact
keys = append(keys, docsObjectKey(namespace, name, version, rest))
// if it looks like a directory or has no extension, try index under it
if strings.HasSuffix(rest, "/") || filepath.Ext(rest) == "" {
keys = append(keys, docsObjectKey(namespace, name, version, strings.TrimSuffix(rest, "/")+"/index.html"))
}
// finally, try root index
keys = append(keys, docsObjectKey(namespace, name, version, "index.html"))
return keys
}
// docsHandler serves static documentation files from S3 (public-facing via Ingress).
func docsHandler(w http.ResponseWriter, r *http.Request) {
ns, name, ver, rest, err := parseDocsRequestPath(r.URL.Path)
if err != nil {
http.Error(w, "Bad docs path", http.StatusBadRequest)
return
}
ctx := context.Background()
candidates := tryCandidateKeys(ns, name, ver, rest)
log.Printf("Docs candidates (host=%s): %v", hostname, candidates)
var lastErr error
for _, key := range candidates {
obj, err := s3Client.GetObject(ctx, bucketName, key, s3.GetObjectOptions{})
if err != nil {
lastErr = err
continue
}
stat, err := obj.Stat()
if err != nil {
lastErr = err
_ = obj.Close()
continue
}
// Determine content-type
ext := filepath.Ext(key)
ctype := mime.TypeByExtension(ext)
if ctype == "" {
// fallback for HTML
if ext == ".html" || strings.HasSuffix(key, "index.html") {
ctype = "text/html; charset=utf-8"
} else {
ctype = "application/octet-stream"
}
}
w.Header().Set("Content-Type", ctype)
w.Header().Set("Content-Length", fmt.Sprintf("%d", stat.Size))
w.Header().Set("Last-Modified", stat.LastModified.Format(http.TimeFormat))
if _, err := io.Copy(w, obj); err != nil {
log.Printf("Error streaming object %s: %v", key, err)
}
_ = obj.Close()
return
}
log.Printf("Docs not found in candidates: %v, lastErr: %v", candidates, lastErr)
http.Error(w, "Documentation not found", http.StatusNotFound)
}
server/gpg_key.go
+8
View File
@@ -0,0 +1,8 @@
package main
import registrykeys "terraform-registry-keys"
var gpgPublicKey = GPGPublicKey{
KeyID: registrykeys.KeyID,
ASCIIArmor: registrykeys.ASCIIArmor,
}
server/main.go
+288
View File
@@ -0,0 +1,288 @@
package main
import (
"bufio"
"context"
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"net/url"
"os"
"strings"
s3 "github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
)
var (
s3Client *s3.Client
bucketName = "terraform-registry" // Default
hostname = os.Getenv("REGISTRY_HOSTNAME")
)
// Terraform Registry Protocol Structs
type Discovery struct {
ProvidersV1 string `json:"providers.v1"`
}
type VersionList struct {
ID string `json:"id"`
Versions []Version `json:"versions"`
Warnings []string `json:"warnings"`
}
type Version struct {
Version string `json:"version"`
Protocols []string `json:"protocols"`
Platforms []Platform `json:"platforms"`
}
type Platform struct {
OS string `json:"os"`
Arch string `json:"arch"`
}
type DownloadResponse struct {
Protocols []string `json:"protocols"`
OS string `json:"os"`
Arch string `json:"arch"`
Filename string `json:"filename"`
DownloadURL string `json:"download_url"`
ShasumsURL string `json:"shasums_url"`
ShasumsSignatureURL string `json:"shasums_signature_url"`
Shasum string `json:"shasum"`
SigningKeys SigningKeys `json:"signing_keys"`
}
type SigningKeys struct {
GPGPublicKeys []GPGPublicKey `json:"gpg_public_keys"`
}
type GPGPublicKey struct {
KeyID string `json:"key_id"`
ASCIIArmor string `json:"ascii_armor"`
}
func main() {
// 1. Init S3 Connection
if hostname == "" {
hostname = "localhost:8080"
}
endpoint := os.Getenv("S3_ENDPOINT")
accessKeyID := os.Getenv("S3_ACCESS_KEY")
secretAccessKey := os.Getenv("S3_SECRET_KEY")
if os.Getenv("S3_BUCKET") != "" {
bucketName = os.Getenv("S3_BUCKET")
}
var err error
s3Client, err = s3.New(endpoint, &s3.Options{
Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
Secure: true, // Force secure for cloud S3
})
if err != nil {
log.Fatalln(err)
}
// 2. HTTP Handlers
http.HandleFunc("/.well-known/terraform.json", discoveryHandler)
http.HandleFunc("/v1/providers/", router)
http.HandleFunc("/v1/proxy", proxyHandler)
http.HandleFunc("/docs/", docsHandler)
http.Handle("/", http.HandlerFunc(rootHandler))
http.HandleFunc("/healthz", healthzHandler)
http.HandleFunc("/readyz", readyzHandler)
log.Printf("Starting Registry Service on :8080 (Bucket: %s, Endpoint: %s)\n", bucketName, endpoint)
log.Fatal(http.ListenAndServe(":8080", nil))
}
func rootHandler(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/" {
http.NotFound(w, r)
return
}
w.Header().Set("Content-Type", "text/html; charset=utf-8")
fmt.Fprint(w, `
<!DOCTYPE html>
<html>
<head><title>Terra Registry</title></head>
<body>
<h1>Terra Registry & Documentation Server</h1>
<p>Status: <span style="color: green">ONLINE</span></p>
<hr>
<p>Powered by Nubes Cloud S3 Storage</p>
</body>
</html>
`)
}
func proxyHandler(w http.ResponseWriter, r *http.Request) {
bucket := r.URL.Query().Get("bucket")
key := r.URL.Query().Get("key")
if bucket == "" || key == "" {
http.Error(w, "Missing bucket or key params", http.StatusBadRequest)
return
}
obj, err := s3Client.GetObject(context.Background(), bucket, key, s3.GetObjectOptions{})
if err != nil {
log.Printf("Error getting object %s/%s: %v", bucket, key, err)
http.Error(w, "File not found", http.StatusNotFound)
return
}
defer obj.Close()
stat, err := obj.Stat()
if err != nil {
log.Printf("Error stating object %s/%s: %v", bucket, key, err)
http.Error(w, "File not found or not accessible", http.StatusNotFound)
return
}
w.Header().Set("Content-Length", fmt.Sprintf("%d", stat.Size))
w.Header().Set("Content-Type", "application/octet-stream")
w.Header().Set("Last-Modified", stat.LastModified.Format(http.TimeFormat))
if _, err := io.Copy(w, obj); err != nil {
log.Printf("Error streaming object: %v", err)
}
}
func discoveryHandler(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(Discovery{ProvidersV1: "/v1/providers/"})
}
func router(w http.ResponseWriter, r *http.Request) {
path := strings.TrimPrefix(r.URL.Path, "/v1/providers/")
parts := strings.Split(path, "/")
if len(parts) == 3 && parts[2] == "versions" {
listVersions(w, r, parts[0], parts[1])
return
}
if len(parts) == 6 && parts[3] == "download" {
downloadVersion(w, r, parts[0], parts[1], parts[2], parts[4], parts[5])
return
}
http.Error(w, "Not Found", http.StatusNotFound)
}
func listVersions(w http.ResponseWriter, r *http.Request, namespace, pType string) {
prefix := fmt.Sprintf("%s/%s/%s/", hostname, namespace, pType)
ctx := context.Background()
versions := []Version{}
seenVersions := map[string]*Version{}
objectCh := s3Client.ListObjects(ctx, bucketName, s3.ListObjectsOptions{
Prefix: prefix,
Recursive: true,
})
for object := range objectCh {
if object.Err != nil {
continue
}
parts := strings.Split(object.Key, "/")
if len(parts) < 5 {
continue
}
verStr := parts[3]
fileName := parts[4]
if _, ok := seenVersions[verStr]; !ok {
seenVersions[verStr] = &Version{
Version: verStr,
Protocols: []string{"5.0"},
Platforms: []Platform{},
}
}
if strings.Contains(fileName, "_linux_amd64.zip") {
seenVersions[verStr].Platforms = append(seenVersions[verStr].Platforms, Platform{OS: "linux", Arch: "amd64"})
}
if strings.Contains(fileName, "_windows_amd64.zip") {
seenVersions[verStr].Platforms = append(seenVersions[verStr].Platforms, Platform{OS: "windows", Arch: "amd64"})
}
}
for _, v := range seenVersions {
versions = append(versions, *v)
}
resp := VersionList{
ID: fmt.Sprintf("%s/%s", namespace, pType),
Versions: versions,
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(resp)
}
func downloadVersion(w http.ResponseWriter, r *http.Request, namespace, pType, version, osType, arch string) {
basePath := fmt.Sprintf("%s/%s/%s/%s", hostname, namespace, pType, version)
filename := fmt.Sprintf("terraform-provider-%s_%s_%s_%s.zip", pType, version, osType, arch)
fullKey := fmt.Sprintf("%s/%s", basePath, filename)
shasumsKey := fmt.Sprintf("%s/terraform-provider-%s_%s_SHA256SUMS", basePath, pType, version)
sigKey := fmt.Sprintf("%s/terraform-provider-%s_%s_SHA256SUMS.sig", basePath, pType, version)
var shasumValue string
shasumsObj, err := s3Client.GetObject(context.Background(), bucketName, shasumsKey, s3.GetObjectOptions{})
if err == nil {
defer shasumsObj.Close()
scanner := bufio.NewScanner(shasumsObj)
for scanner.Scan() {
line := scanner.Text()
if strings.Contains(line, filename) {
fields := strings.Fields(line)
if len(fields) >= 1 {
shasumValue = fields[0]
}
break
}
}
}
baseURL := "https://" + hostname
downloadLink := fmt.Sprintf("%s/v1/proxy?bucket=%s&key=%s", baseURL, bucketName, url.QueryEscape(fullKey))
shasumsLink := fmt.Sprintf("%s/v1/proxy?bucket=%s&key=%s", baseURL, bucketName, url.QueryEscape(shasumsKey))
sigLink := fmt.Sprintf("%s/v1/proxy?bucket=%s&key=%s", baseURL, bucketName, url.QueryEscape(sigKey))
gpgKey := gpgPublicKey
resp := DownloadResponse{
Protocols: []string{"5.0"},
OS: osType,
Arch: arch,
Filename: filename,
DownloadURL: downloadLink,
ShasumsURL: shasumsLink,
ShasumsSignatureURL: sigLink,
Shasum: shasumValue,
SigningKeys: SigningKeys{
GPGPublicKeys: []GPGPublicKey{gpgKey},
},
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(resp)
}
func healthzHandler(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
w.Write([]byte("ok"))
}
func readyzHandler(w http.ResponseWriter, r *http.Request) {
// Проверить доступность S3
w.WriteHeader(http.StatusOK)
w.Write([]byte("ok"))
}