WIP parca debug-info upload

go.mod

@@ -5,6 +5,8 @@ go 1.24.6
 toolchain go1.24.9
 
 require (
+	buf.build/gen/go/parca-dev/parca/grpc/go v1.5.1-20250212095114-4db6f2d46517.2
+	buf.build/gen/go/parca-dev/parca/protocolbuffers/go v1.36.6-20250212095114-4db6f2d46517.1
 	connectrpc.com/connect v1.19.1
 	connectrpc.com/grpchealth v1.4.0
 	github.com/PuerkitoBio/goquery v1.10.3
@@ -100,12 +102,18 @@ require (
 	google.golang.org/grpc v1.76.0
 	google.golang.org/protobuf v1.36.10
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
+	gopkg.in/dnaeon/go-vcr.v3 v3.2.2
 	gopkg.in/yaml.v3 v3.0.1
 	sigs.k8s.io/yaml v1.6.0
 )
 
 require (
 	github.com/Masterminds/semver/v3 v3.4.0 // indirect
+	github.com/baidubce/bce-sdk-go v0.9.111 // indirect
+	github.com/gofrs/flock v0.8.1 // indirect
+	github.com/huaweicloud/huaweicloud-sdk-go-obs v3.25.4+incompatible // indirect
+	github.com/oracle/oci-go-sdk/v65 v65.41.1 // indirect
+	github.com/sony/gobreaker v0.5.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.2 // indirect
 )
 
@@ -298,7 +306,7 @@ require (
 	go.opentelemetry.io/contrib/detectors/gcp v1.36.0 // indirect
 	go.opentelemetry.io/contrib/exporters/autoexport v0.61.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.61.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.61.0
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect
 	go.opentelemetry.io/contrib/propagators/jaeger v1.35.0 // indirect
 	go.opentelemetry.io/contrib/samplers/jaegerremote v0.30.0 // indirect
@@ -319,7 +327,7 @@ require (
 	go.opentelemetry.io/otel/sdk v1.37.0 // indirect
 	go.opentelemetry.io/otel/sdk/log v0.12.2 // indirect
 	go.opentelemetry.io/otel/sdk/metric v1.37.0 // indirect
-	go.opentelemetry.io/otel/trace v1.38.0 // indirect
+	go.opentelemetry.io/otel/trace v1.38.0
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
 	golang.org/x/crypto v0.45.0 // indirect

go.sum

@@ -1,3 +1,7 @@
+buf.build/gen/go/parca-dev/parca/grpc/go v1.5.1-20250212095114-4db6f2d46517.2 h1:AiTEEAQ8AHE5udDizcA2r3GNjxxlk4f2Buot/hA2oYM=
+buf.build/gen/go/parca-dev/parca/grpc/go v1.5.1-20250212095114-4db6f2d46517.2/go.mod h1:U8BtFPtz71GSALR7K7ALn39RnvrKsD++lJzMI5Gf4fs=
+buf.build/gen/go/parca-dev/parca/protocolbuffers/go v1.36.6-20250212095114-4db6f2d46517.1 h1:gXHJuGlWoXUkH9O9Qxw9skNAhIDgNgJx0tYSbI8fjIo=
+buf.build/gen/go/parca-dev/parca/protocolbuffers/go v1.36.6-20250212095114-4db6f2d46517.1/go.mod h1:58eXMQL4tavOGzyXIveWU4f3yTFGmLIHxM9uk11DOIo=
 cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY=
 cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw=
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
@@ -767,6 +771,7 @@ github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
@@ -1170,6 +1175,8 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/dnaeon/go-vcr.v3 v3.2.2 h1:bYA+2JpMNv9TVYS5Z49o8YjB0OVX+vpL4Yf4mkdPNAc=
+gopkg.in/dnaeon/go-vcr.v3 v3.2.2/go.mod h1:Oy/wiaJrF0S8WmwjGRpDSL09yGnb9HXV4qP87dHkfFk=
 gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4=
 gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=

go.work.sum

@@ -1,3 +1,5 @@
+buf.build/gen/go/parca-dev/parca/protocolbuffers/go v1.36.6-20250212095114-4db6f2d46517.1 h1:gXHJuGlWoXUkH9O9Qxw9skNAhIDgNgJx0tYSbI8fjIo=
+buf.build/gen/go/parca-dev/parca/protocolbuffers/go v1.36.6-20250212095114-4db6f2d46517.1/go.mod h1:58eXMQL4tavOGzyXIveWU4f3yTFGmLIHxM9uk11DOIo=
 cloud.google.com/go/accessapproval v1.8.6 h1:UkmDPCKvj24bkGVrvgJPcgSDkmIPw/bAmOiDb9avOiE=
 cloud.google.com/go/accessapproval v1.8.6/go.mod h1:FfmTs7Emex5UvfnnpMkhuNkRCP85URnBFt5ClLxhZaQ=
 cloud.google.com/go/accesscontextmanager v1.9.6 h1:2LnncRqfYB8NEdh9+FeYxAt9POTW/0zVboktnRlO11w=

lidia/lidia.go

@@ -7,6 +7,7 @@ package lidia
 
 import (
 	"debug/elf"
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -163,7 +164,7 @@ func CreateLidiaFromELF(elfFile *elf.File, output io.WriteSeeker, opts ...Option
 	}
 
 	symbols, err := elfFile.Symbols()
-	if err != nil {
+	if err != nil && !errors.Is(err, elf.ErrNoSymbols) {
 		return fmt.Errorf("failed to read symbols from ELF file: %w", err)
 	}
 

pkg/api/api.go

@@ -12,13 +12,13 @@ import (
 	"net/http"
 
 	"connectrpc.com/connect"
-
 	"github.com/felixge/fgprof"
 	"github.com/go-kit/log"
 	"github.com/grafana/dskit/kv/memberlist"
 	"github.com/grafana/dskit/middleware"
 	"github.com/grafana/dskit/server"
 	grpcgw "github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
+	"google.golang.org/grpc"
 
 	"github.com/grafana/pyroscope/public"
 
@@ -190,6 +190,21 @@ func (a *API) RegisterOverridesExporter(oe *exporter.OverridesExporter) {
 	})
 }
 
+func (a *API) RegisterDebugInfo(d *grpc.Server, limits *validation.Overrides) {
+	writePathOpts := a.registerOptionsWritePath(limits)
+	const (
+		DebuginfoService_Upload_FullMethodName               = "/parca.debuginfo.v1alpha1.DebuginfoService/Upload"
+		DebuginfoService_ShouldInitiateUpload_FullMethodName = "/parca.debuginfo.v1alpha1.DebuginfoService/ShouldInitiateUpload"
+		DebuginfoService_InitiateUpload_FullMethodName       = "/parca.debuginfo.v1alpha1.DebuginfoService/InitiateUpload"
+		DebuginfoService_MarkUploadFinished_FullMethodName   = "/parca.debuginfo.v1alpha1.DebuginfoService/MarkUploadFinished"
+	)
+
+	a.RegisterRoute(DebuginfoService_Upload_FullMethodName, d, writePathOpts...)
+	a.RegisterRoute(DebuginfoService_ShouldInitiateUpload_FullMethodName, d, writePathOpts...)
+	a.RegisterRoute(DebuginfoService_InitiateUpload_FullMethodName, d, writePathOpts...)
+	a.RegisterRoute(DebuginfoService_MarkUploadFinished_FullMethodName, d, writePathOpts...)
+}
+
 // RegisterDistributor registers the endpoints associated with the distributor.
 func (a *API) RegisterDistributor(d *distributor.Distributor, limits *validation.Overrides, cfg server.Config) {
 	writePathOpts := a.registerOptionsWritePath(limits)
@@ -206,6 +221,7 @@ func (a *API) RegisterDistributor(d *distributor.Distributor, limits *validation
 
 	a.RegisterRoute("/opentelemetry.proto.collector.profiles.v1development.ProfilesService/Export", otlpHandler, writePathOpts...)
 	a.RegisterRoute("/v1development/profiles", otlpHandler, writePathOpts...)
+
 }
 
 // RegisterMemberlistKV registers the endpoints associated with the memberlist KV store.

pkg/debuginfo/parcaglue.go

@@ -0,0 +1,61 @@
+package debuginfo
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"time"
+
+	"buf.build/gen/go/parca-dev/parca/grpc/go/parca/debuginfo/v1alpha1/debuginfov1alpha1grpc"
+	"github.com/go-kit/log"
+	"go.opentelemetry.io/otel/trace/noop"
+	"google.golang.org/grpc"
+
+	"github.com/grafana/pyroscope/pkg/objstore"
+	"github.com/grafana/pyroscope/pkg/parca/debuginfo"
+	"github.com/grafana/pyroscope/pkg/symbolizer"
+
+	debuginfopb "buf.build/gen/go/parca-dev/parca/protocolbuffers/go/parca/debuginfo/v1alpha1"
+)
+
+type Config struct {
+	Enabled           bool          `yaml:"-"`
+	MaxUploadSize     int64         `yaml:"-"`
+	MaxUploadDuration time.Duration `yaml:"-"`
+}
+
+func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
+	f.BoolVar(&cfg.Enabled, "debug-info.enabled", true, "Enable debug info.")
+	f.Int64Var(&cfg.MaxUploadSize, "debug-info.max-upload-size", 100*1024*1024, "Maximum size of a single debug info upload in bytes.")
+	f.DurationVar(&cfg.MaxUploadDuration, "debug-info.max-upload-duration", time.Minute, "Maximum duration of a single debug info upload.")
+}
+
+func NewParcaDebugInfo(l log.Logger, bucket objstore.Bucket, cfg Config, server *grpc.Server) error {
+	if !cfg.Enabled {
+		debuginfov1alpha1grpc.RegisterDebuginfoServiceServer(server, disabled{})
+		return nil
+	}
+	if bucket == nil {
+		return fmt.Errorf("storage bucket is required for debug-info")
+	}
+
+	t := noop.Tracer{}
+	l = log.With(l, "component", "debug-info")
+	bucket = objstore.NewPrefixedBucket(bucket, symbolizer.BucketPrefixParcaDebugInfo)
+	md := debuginfo.NewObjectStoreMetadata(l, bucket)
+	debuginfod := debuginfo.NewParallelDebuginfodClients(nil)
+	store, _ := debuginfo.NewStore(t, l, md, bucket, debuginfod,
+		debuginfo.SignedUpload{Enabled: false},
+		cfg.MaxUploadDuration, cfg.MaxUploadSize)
+
+	debuginfov1alpha1grpc.RegisterDebuginfoServiceServer(server, store)
+	return nil
+}
+
+type disabled struct {
+	debuginfov1alpha1grpc.UnimplementedDebuginfoServiceServer
+}
+
+func (d disabled) ShouldInitiateUpload(context.Context, *debuginfopb.ShouldInitiateUploadRequest) (*debuginfopb.ShouldInitiateUploadResponse, error) {
+	return &debuginfopb.ShouldInitiateUploadResponse{ShouldInitiateUpload: false, Reason: "debug info upload is disabled"}, nil
+}

pkg/ingester/otlp/ingest_handler.go

@@ -8,12 +8,12 @@ import (
 	"net/http"
 	"strings"
 
-	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
-	"google.golang.org/grpc/keepalive"
 	"google.golang.org/protobuf/encoding/protojson"
 	"google.golang.org/protobuf/proto"
 
+	"github.com/grafana/pyroscope/pkg/util/httpgrpc"
+
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	"github.com/google/uuid"
@@ -52,7 +52,7 @@ func NewOTLPIngestHandler(cfg server.Config, svc PushService, l log.Logger) Hand
 		log: l,
 	}
 
-	grpcServer := newGrpcServer(cfg)
+	grpcServer := httpgrpc.NewGrpcServer(cfg)
 	pprofileotlp.RegisterProfilesServiceServer(grpcServer, h)
 
 	h.handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -74,34 +74,6 @@ func NewOTLPIngestHandler(cfg server.Config, svc PushService, l log.Logger) Hand
 	return h
 }
 
-func newGrpcServer(cfg server.Config) *grpc.Server {
-	grpcKeepAliveOptions := keepalive.ServerParameters{
-		MaxConnectionIdle:     cfg.GRPCServerMaxConnectionIdle,
-		MaxConnectionAge:      cfg.GRPCServerMaxConnectionAge,
-		MaxConnectionAgeGrace: cfg.GRPCServerMaxConnectionAgeGrace,
-		Time:                  cfg.GRPCServerTime,
-		Timeout:               cfg.GRPCServerTimeout,
-	}
-
-	grpcKeepAliveEnforcementPolicy := keepalive.EnforcementPolicy{
-		MinTime:             cfg.GRPCServerMinTimeBetweenPings,
-		PermitWithoutStream: cfg.GRPCServerPingWithoutStreamAllowed,
-	}
-
-	grpcOptions := []grpc.ServerOption{
-		grpc.KeepaliveParams(grpcKeepAliveOptions),
-		grpc.KeepaliveEnforcementPolicy(grpcKeepAliveEnforcementPolicy),
-		grpc.MaxRecvMsgSize(cfg.GRPCServerMaxRecvMsgSize),
-		grpc.MaxSendMsgSize(cfg.GRPCServerMaxSendMsgSize),
-		grpc.MaxConcurrentStreams(uint32(cfg.GRPCServerMaxConcurrentStreams)),
-		grpc.NumStreamWorkers(uint32(cfg.GRPCServerNumWorkers)),
-	}
-
-	grpcOptions = append(grpcOptions, cfg.GRPCOptions...)
-
-	return grpc.NewServer(grpcOptions...)
-}
-
 func (h *ingestHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	h.handler.ServeHTTP(w, r)
 }

pkg/parca/cache/cache.go

@@ -0,0 +1,73 @@
+// Copyright 2023-2025 The Parca Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cache
+
+import (
+	"sync"
+
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/grafana/pyroscope/pkg/parca/cache/lru"
+)
+
+type LRUCache[K comparable, V any] struct {
+	lru *lru.LRU[K, V]
+	mtx *sync.RWMutex
+}
+
+func NewLRUCache[K comparable, V any](reg prometheus.Registerer, maxEntries int) *LRUCache[K, V] {
+	return &LRUCache[K, V]{
+		lru: lru.New[K, V](reg, lru.WithMaxSize[K, V](maxEntries)),
+		mtx: &sync.RWMutex{},
+	}
+}
+
+func (c *LRUCache[K, V]) Add(key K, value V) {
+	c.mtx.Lock()
+	defer c.mtx.Unlock()
+	c.lru.Add(key, value)
+}
+
+func (c *LRUCache[K, V]) Get(key K) (V, bool) {
+	c.mtx.RLock()
+	defer c.mtx.RUnlock()
+	return c.lru.Get(key)
+}
+
+// Peek returns the value associated with key without updating the "recently used"-ness of that key.
+func (c *LRUCache[K, V]) Peek(key K) (V, bool) {
+	c.mtx.RLock()
+	defer c.mtx.RUnlock()
+	return c.lru.Peek(key)
+}
+
+func (c *LRUCache[K, V]) Remove(key K) {
+	c.mtx.Lock()
+	defer c.mtx.Unlock()
+	c.lru.Remove(key)
+}
+
+func (c *LRUCache[K, V]) Purge() {
+	c.mtx.Lock()
+	defer c.mtx.Unlock()
+	c.lru.Purge()
+}
+
+func (c *LRUCache[K, V]) Close() error {
+	c.mtx.Lock()
+	defer c.mtx.Unlock()
+
+	c.lru.Purge()
+	return c.lru.Close()
+}