Skip to content

Instantly share code, notes, and snippets.

@sttts
Created July 24, 2023 14:15
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save sttts/dea306ccfef35fac2f8ba692b622ea8e to your computer and use it in GitHub Desktop.
Save sttts/dea306ccfef35fac2f8ba692b622ea8e to your computer and use it in GitHub Desktop.
diff --git a/internal/plugin/grpc_provider.go b/internal/plugin/grpc_provider.go
index e79fcaa2df..182b148fc6 100644
--- a/internal/plugin/grpc_provider.go
+++ b/internal/plugin/grpc_provider.go
@@ -7,7 +7,9 @@ import (
"context"
"errors"
"fmt"
+ "os"
"sync"
+ "time"
"github.com/zclconf/go-cty/cty"
@@ -20,6 +22,7 @@ import (
ctyjson "github.com/zclconf/go-cty/cty/json"
"github.com/zclconf/go-cty/cty/msgpack"
"google.golang.org/grpc"
+ goproto "google.golang.org/protobuf/proto"
)
var logger = logging.HCLogger()
@@ -79,8 +82,8 @@ func (p *GRPCProvider) GetProviderSchema() (resp providers.GetProviderSchemaResp
defer p.mu.Unlock()
// check the global cache if we can
- if !p.Addr.IsZero() && resp.ServerCapabilities.GetProviderSchemaOptional {
- if resp, ok := providers.SchemaCache.Get(p.Addr); ok {
+ if !p.Addr.IsZero() {
+ if resp, ok := providers.SchemaCache.Get(p.Addr); ok && resp.ServerCapabilities.GetProviderSchemaOptional {
return resp
}
}
@@ -94,6 +97,33 @@ func (p *GRPCProvider) GetProviderSchema() (resp providers.GetProviderSchemaResp
resp.ResourceTypes = make(map[string]providers.Schema)
resp.DataSources = make(map[string]providers.Schema)
+ var protoResp *proto.GetProviderSchema_Response
+ cacheFileName := "provider-getproviderschema.bin"
+ cacheAddr := p.Addr
+ if cacheAddr.IsZero() {
+ cacheAddr = addrs.NewBuiltInProvider("fake-shouldnt-be-constant")
+ }
+ if os.Getenv("TF_CACHE") == "1" {
+ startRead := time.Now()
+ if resp, ok := providers.SchemaCache.Get(cacheAddr); ok {
+ logger.Info("Using cached schema", "file", "in-memory")
+ return resp
+ }
+ if bs, err := os.ReadFile(cacheFileName); err == nil {
+ logger.Info("Using cached schema", "file", cacheFileName)
+ startUnmarshal := time.Now()
+ protoResp = new(proto.GetProviderSchema_Response)
+ err := goproto.Unmarshal(bs, protoResp)
+ logger.Info("GetProviderSchema Unmarshal from cache", "duration", time.Since(startUnmarshal))
+ logger.Info("GetProviderSchema ReadFile from cache", "duration", time.Since(startRead))
+ if err != nil {
+ protoResp = nil
+ logger.Warn("failed to unmarshal cached schema", "error", err)
+ // fall-through
+ }
+ }
+ }
+
// Some providers may generate quite large schemas, and the internal default
// grpc response size limit is 4MB. 64MB should cover most any use case, and
// if we get providers nearing that we may want to consider a finer-grained
@@ -102,17 +132,32 @@ func (p *GRPCProvider) GetProviderSchema() (resp providers.GetProviderSchemaResp
// this for compatibility, but recent providers all set the max message
// size much higher on the server side, which is the supported method for
// determining payload size.
- const maxRecvSize = 64 << 20
- protoResp, err := p.client.GetSchema(p.ctx, new(proto.GetProviderSchema_Request), grpc.MaxRecvMsgSizeCallOption{MaxRecvMsgSize: maxRecvSize})
- if err != nil {
- resp.Diagnostics = resp.Diagnostics.Append(grpcErr(err))
- return resp
- }
+ if protoResp == nil {
+ const maxRecvSize = 64 << 20
+ var err error
+ start := time.Now()
+ protoResp, err = p.client.GetSchema(p.ctx, new(proto.GetProviderSchema_Request), grpc.MaxRecvMsgSizeCallOption{MaxRecvMsgSize: maxRecvSize})
+ logger.Info("GetProviderSchema", "duration", time.Since(start))
+ if err != nil {
+ resp.Diagnostics = resp.Diagnostics.Append(grpcErr(err))
+ return resp
+ }
- resp.Diagnostics = resp.Diagnostics.Append(convert.ProtoToDiagnostics(protoResp.Diagnostics))
+ if !resp.Diagnostics.HasErrors() && os.Getenv("TF_CACHE") == "1" {
+ bs, err := goproto.Marshal(protoResp)
+ if err == nil {
+ if err := os.WriteFile(cacheFileName, bs, 0644); err != nil {
+ logger.Warn("failed to write schema cache file", "error", err)
+ // fall-through
+ }
+ }
+ }
- if resp.Diagnostics.HasErrors() {
- return resp
+ resp.Diagnostics = resp.Diagnostics.Append(convert.ProtoToDiagnostics(protoResp.Diagnostics))
+
+ if resp.Diagnostics.HasErrors() {
+ return resp
+ }
}
if protoResp.Provider == nil {
@@ -143,6 +188,8 @@ func (p *GRPCProvider) GetProviderSchema() (resp providers.GetProviderSchemaResp
// set the global cache if we can
if !p.Addr.IsZero() {
providers.SchemaCache.Set(p.Addr, resp)
+ } else if _, ok := providers.SchemaCache.Get(cacheAddr); !ok {
+ providers.SchemaCache.Set(cacheAddr, resp)
}
// always store this here in the client for providers that are not able to
diff --git a/main.go b/main.go
index 075c335687..73204192a5 100644
--- a/main.go
+++ b/main.go
@@ -13,6 +13,7 @@ import (
"path/filepath"
"runtime"
"strings"
+ "time"
"github.com/apparentlymart/go-shquot/shquot"
"github.com/hashicorp/go-plugin"
@@ -67,6 +68,11 @@ func main() {
func realMain() int {
defer logging.PanicHandler()
+ start := time.Now()
+ defer func(start time.Time) {
+ fmt.Printf("[INFO] Terraform exited in %s.\n", time.Since(start))
+ }(start)
+
var err error
err = openTelemetryInit()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment