istio/envoy v8 flags

1-exploration.md

exploring profiling wasm/v8

local repo paths

1. istio/proxy: /home/oschaaf/code/istio/proxy
2. istio/envoy: /home/oschaaf/code/istio/envoy -- ^^ uses this, overriden in userbazelrc
3. v8: /home/oschaaf/code/v8/v8 -> for code exploration only. not build/linked

Build istio envoy & wasm plugins with the diffs supplied below

 CC=/opt/llvm/bin/clang CPP=/opt/llvm/bin/clang++ PATH="/opt/llvm/bin:$PATH" make build_envoy
 CC=/opt/llvm/bin/clang CPP=/opt/llvm/bin/clang++ PATH="/opt/llvm/bin:$PATH" make build_wasm

run envoy & obtain profile (perf.data)

cd ~/code/istio/proxy

V8OPTS="--perf_basic_prof --perf_prof --perf_prof_annotate_wasm" perf record -e cycles:u -g  -k mono -- bazel-out/k8-opt/bin/src/envoy/envoy --config-
path envoy.yaml --concurrency 1

# alt, based on some tidbits I read here and there:
# --callgraph fp, because dwarf might not work well with jit

V8OPTS="--perf_prof --nowrite-protect-code-memory -nowasm_write_protect_code_memory" perf record -R --call-graph fp -e cycles:u -g  -k mono -- bazel-out/k8-opt/bin/src/envoy/envoy --config-path envoy.yaml --concurrency 1

# note: envoy ought to be running and going through a wasm plugin at this address
curl -vv 127.0.0.1:8081
# note: admin interface here. e.g. enable built-in cpu profiler. note -- we don't use that in this flow.
curl -X POST http://localhost:8002/cpuprofiler?enable=y

# injecting jitted data. takes a long time. will produce .so files, which don't help as they have the same unknown turbo symbols.
perf inject -j -i perf.data -o perf.data.jitted
 
# sometimes you'll end up with more .so files then you can handle. this cleans it up.
find . -maxdepth 1 -name "jitted-*-*.so" -print0 | xargs -0 rm

produce flamegraph from perf.data

# note -- use perf.data.jitted if you used the perf inject line from above.  
perf script -i perf.data |  ~/tmp/FlameGraph/stackcollapse-perf.pl > out.perf-folded && ~/tmp/FlameGraph/flamegraph.pl out.perf-folded > node-flame.svg

envoy.yaml

node:
  id: test-server-ratings
  metadata:
    ISTIO_VERSION: "1.4-dev"
    EXCHANGE_KEYS: "NAME,NAMESPACE,INSTANCE_IPS,LABELS,OWNER,PLATFORM_METADATA,WORKLOAD_NAME"
    NAME: ratings-v22-84975bc778-pxz2w
    NAMESPACE: default
    WORKLOAD_NAME: ratings
    OWNER: /api/ns/deployment/ratings-deployment
    LABELS: { app: ratings, version: V22 }
    INSTANCE_IPS: "10.52.0.35,fe80::a075:11ff:fe5e:f1cd"
    istio: sidecar
    PLATFORM_METADATA:
      gcp_cluster_name: /redacted/
      gcp_project: "/redacted/"
      gcp_cluster_location: "us-east4-b"
stats_config:
  use_all_default_tags: false
  stats_tags:
  - tag_name: "reporter"
    regex: "(reporter=\\.=(.+?);\\.;)"
  - tag_name: "source_namespace"
    regex: "(source_namespace=\\.=(.+?);\\.;)"
  - tag_name: "source_workload"
    regex: "(source_workload=\\.=(.+?);\\.;)"
  - tag_name: "source_canonical_service"
    regex: "(source_canonical_service=\\.=(.+?);\\.;)"
  - tag_name: "source_canonical_revision"
    regex: "(source_canonical_revision=\\.=(.*?);\\.;)"
  - tag_name: "source_workload_namespace"
    regex: "(source_workload_namespace=\\.=(.+?);\\.;)"
  - tag_name: "source_principal"
    regex: "(source_principal=\\.=(.+?);\\.;)"
  - tag_name: "source_app"
    regex: "(source_app=\\.=(.+?);\\.;)"
  - tag_name: "source_version"
    regex: "(source_version=\\.=(.+?);\\.;)"
  - tag_name: "destination_namespace"
    regex: "(destination_namespace=\\.=(.+?);\\.;)"
  - tag_name: "destination_workload"
    regex: "(destination_workload=\\.=(.+?);\\.;)"
  - tag_name: "destination_workload_namespace"
    regex: "(destination_workload_namespace=\\.=(.+?);\\.;)"
  - tag_name: "destination_principal"
    regex: "(destination_principal=\\.=(.+?);\\.;)"
  - tag_name: "destination_app"
    regex: "(destination_app=\\.=(.+?);\\.;)"
  - tag_name: "destination_version"
    regex: "(destination_version=\\.=(.+?);\\.;)"
  - tag_name: "destination_service"
    regex: "(destination_service=\\.=(.+?);\\.;)"
  - tag_name: "destination_canonical_service"
    regex: "(destination_canonical_service=\\.=(.+?);\\.;)"
  - tag_name: "destination_canonical_revision"
    regex: "(destination_canonical_revision=\\.=(.*?);\\.;)"
  - tag_name: "destination_service_name"
    regex: "(destination_service_name=\\.=(.+?);\\.;)"
  - tag_name: "destination_service_namespace"
    regex: "(destination_service_namespace=\\.=(.+?);\\.;)"
  - tag_name: "request_protocol"
    regex: "(request_protocol=\\.=(.+?);\\.;)"
  - tag_name: "request_operation"
    regex: "(request_operation=\\.=(.+?);\\.;)"
  - tag_name: "response_code"
    regex: "(response_code=\\.=(.+?);\\.;)|_rq(_(\\.d{3}))$"
  - tag_name: "grpc_response_status"
    regex: "(grpc_response_status=\\.=(.*?);\\.;)"
  - tag_name: "response_flags"
    regex: "(response_flags=\\.=(.+?);\\.;)"
  - tag_name: "connection_security_policy"
    regex: "(connection_security_policy=\\.=(.+?);\\.;)"
  - tag_name: "cache"
    regex: "(cache\\.(.+?)\\.)"
  - tag_name: "component"
    regex: "(component\\.(.+?)\\.)"
  - tag_name: "tag"
    regex: "(tag\\.(.+?);\\.)"
static_resources:
  listeners:
  - name: server
    traffic_direction: INBOUND
    address:
      socket_address:
        address: 0.0.0.0
        port_value: 8081
    filter_chains:
    - filters:
      - name: envoy.http_connection_manager
        config:
          stat_prefix: ingress_http
          codec_type: auto
          route_config:
            name: local_route
            virtual_hosts:
            - name: local_service
              domains:
              - "*"
              routes:
              - match:
                  prefix: "/"
                route:
                  cluster: web_service
          http_filters:
          - name: istio.attributegen
            typed_config:
              '@type': type.googleapis.com/udpa.type.v1.TypedStruct
              type_url: type.googleapis.com/envoy.extensions.filters.http.wasm.v3.Wasm
              value:
                config:
                  configuration: |
                    { "debug": "true",
                      "attributes": [
                        {
                          "output_attribute": "istio_operationId",
                          "match": [
                            {
                              "value": "GetStatus",
                              "condition": "request.url_path.startsWith('/status')"
                            }
                          ]
                        },
                        {
                          "output_attribute": "istio_responseClass",
                          "match": [
                            {
                              "value": "2xx",
                              "condition": "response.code >= 200 && response.code <= 299"
                            }
                          ]
                        }
                      ]
                    }                          
                  vm_config:
                    code:
                      local:
                        inline_string: envoy.wasm.attributegen
                    runtime: envoy.wasm.runtime.null
          - name: istio.metadata_exchange
            typed_config:
              '@type': type.googleapis.com/udpa.type.v1.TypedStruct
              type_url: type.googleapis.com/envoy.extensions.filters.http.wasm.v3.Wasm
              value:
                config:
                  configuration: "{\"name\": \"envoy.wasm.metadata_exchange\"}"
                  vm_config:
                    runtime: "envoy.wasm.runtime.v8"
                    code:
                      local:
                        filename: "extensions/metadata_exchange/plugin.wasm"

                    #code:
                    #  local:
                    #    inline_string: envoy.wasm.metadata_exchange
                    #runtime: envoy.wasm.runtime.null
          - name: istio.stats
            typed_config:
              '@type': type.googleapis.com/udpa.type.v1.TypedStruct
              type_url: type.googleapis.com/envoy.extensions.filters.http.wasm.v3.Wasm
              value:
                config:
                  configuration: |
                    {
                      "debug": "true",
                      "stat_prefix": "istio",
                      "metrics": [
                      {
                        "name": "requests_total",
                        "dimensions": {
                          "request_operation": "istio_operationId"
                         }
                      }
                    ]}
                  root_id: stats_inbound
                  vm_config:
                    runtime: "envoy.wasm.runtime.v8"
                    code:
                      local:
                        filename: "extensions/stats/plugin.wasm"

                    #code:
                    #  local:
                    #    inline_string: envoy.wasm.stats
                    #runtime: envoy.wasm.runtime.null
                    vm_id: stats_inbound
          - name: envoy.router
            config: {}
          access_log:
          - name: envoy.file_access_log
            typed_config:
              '@type': type.googleapis.com/envoy.config.accesslog.v2.FileAccessLog
              path: "/tmp/server1.out"
              format: "server %FILTER_STATE(istio.operationId):20%  error=%FILTER_STATE(istio.operationId_error):20%  origin %RESPONSE_CODE% downstream:%REQ(x-envoy-peer-metadata-id)% downstream:%REQ(x-envoy-peer-metadata)%\n"
  - name: staticreply
    address:
      socket_address:
        address: 127.0.0.1
        port_value: 8099
    filter_chains:
    - filters:
      - name: envoy.http_connection_manager
        config:
          stat_prefix: ingress_http
          codec_type: auto
          route_config:
            name: local_route
            virtual_hosts:
            - name: local_service
              domains:
              - "*"
              routes:
              - match:
                  prefix: "/"
                direct_response:
                  status: 200
                  body:
                    inline_string: "example body\n"
          http_filters:
          - name: envoy.lua
            typed_config:
              "@type": type.googleapis.com/envoy.config.filter.http.lua.v2.Lua
              inline_code: |
                function trivial_httpbin(request_handle)
                  local headers = request_handle:headers()
                  local path = headers:get(":path")
                  local start, _ = path:find("/[^/]*$")
                  local last_segment = path:sub(start+1)

                  request_handle:respond(
                      {[":status"] = last_segment})
                end
                function envoy_on_request(request_handle)
                 trivial_httpbin(request_handle)
                end
          - name: envoy.router
            config: {}
          access_log:
          - name: envoy.file_access_log
            typed_config:
              '@type': type.googleapis.com/envoy.config.accesslog.v2.FileAccessLog
              path: "/tmp/server.out"
              format: "%FILTER_STATE(istio.operationId):20% origin %RESPONSE_CODE% downstream:%REQ(x-envoy-peer-metadata-id)% downstream:%REQ(x-envoy-peer-metadata)%\n"
  clusters:
  - name: web_service
    connect_timeout: 0.25s
    type: static
    lb_policy: round_robin
    hosts:
    - socket_address:
        address: 127.0.0.1
        port_value: 8099
admin:
  access_log_path: "/dev/null"
  profile_path: /tmp/istio/envoy.prof
  address:
    socket_address:
      address: 0.0.0.0
      port_value: 8002

sampleflame.svg

zz istio-proxy diff

diff --git a/.gitignore b/.gitignore
index 9042dcf2..1e761d98 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,3 +17,12 @@ test/envoye2e/http_metadata_exchange/testoutput
 /extensions/common/node_info_bfbs_generated.h
 /extensions/common/proxy_expr.h
 /extensions/common/nlohmann_json.hpp
+*.wasm.map
+*.data
+*.data.jitted
+*.svg
+*.old
+*.diff
+*.gitignore
+*.folded
+
diff --git a/Makefile.core.mk b/Makefile.core.mk
index 73cb7f93..5de4ae8f 100644
--- a/Makefile.core.mk
+++ b/Makefile.core.mk
@@ -30,7 +30,7 @@ endif
 ifeq "$(origin CXX)" "default"
 CXX := clang++
 endif
-PATH := /usr/lib/llvm-9/bin:$(PATH)
+PATH := /usr/lib/llvm/bin:$(PATH)
 
 VERBOSE ?=
 ifeq "$(VERBOSE)" "1"
diff --git a/extensions/attributegen/Makefile b/extensions/attributegen/Makefile
index ec1bd4b4..6353d0db 100644
--- a/extensions/attributegen/Makefile
+++ b/extensions/attributegen/Makefile
@@ -11,7 +11,7 @@ all: plugin.wasm
 
 %.wasm %.wat: %.cc ${CPP_API}/proxy_wasm_intrinsics.h ${CPP_API}/proxy_wasm_enums.h ${CPP_API}/proxy_wasm_externs.h ${CPP_API}/proxy_wasm_api.h ${CPP_API}/proxy_wasm_intrinsics.js ${CPP_CONTEXT_LIB}
 	protoc config.proto --cpp_out=.
-	em++ -s STANDALONE_WASM=1 -s EMIT_EMSCRIPTEN_METADATA=1 -s EXPORTED_FUNCTIONS=['_malloc'] --std=c++17 -O3 -I${CPP_API} -I${CPP_API}/google/protobuf -Iextensions/common -I. -I/usr/local/include -I${ABSL} -I. --js-library ${CPP_API}/proxy_wasm_intrinsics.js $*.cc ${CPP_API}/proxy_wasm_intrinsics.pb.cc ${PROTO_SRCS} ${COMMON_SRCS} ${CPP_CONTEXT_LIB} ${ABSL_CPP} ${CPP_API}/libprotobuf.a -o $*.wasm
+	em++ --profiling -s STANDALONE_WASM=1 -s EMIT_EMSCRIPTEN_METADATA=1 -s EXPORTED_FUNCTIONS=['_malloc'] --std=c++17 -O0 -g4 --profiling-funcs -I${CPP_API} -I${CPP_API}/google/protobuf -Iextensions/common -I. -I/usr/local/include -I${ABSL} -I. --js-library ${CPP_API}/proxy_wasm_intrinsics.js $*.cc ${CPP_API}/proxy_wasm_intrinsics.pb.cc ${PROTO_SRCS} ${COMMON_SRCS} ${CPP_CONTEXT_LIB} ${ABSL_CPP} ${CPP_API}/libprotobuf.a -o $*.wasm
 	rm -f $*.wast
 	rm -f config.pb.*
 	chown ${uid}.${gid} $^
diff --git a/extensions/metadata_exchange/Makefile b/extensions/metadata_exchange/Makefile
index 9c72917c..49dab38d 100644
--- a/extensions/metadata_exchange/Makefile
+++ b/extensions/metadata_exchange/Makefile
@@ -11,7 +11,7 @@ all: plugin.wasm
 
 %.wasm %.wat: %.cc ${CPP_API}/proxy_wasm_intrinsics.h ${CPP_API}/proxy_wasm_enums.h ${CPP_API}/proxy_wasm_externs.h ${CPP_API}/proxy_wasm_api.h ${CPP_API}/proxy_wasm_intrinsics.js ${CPP_CONTEXT_LIB}
 	protoc declare_property.proto --cpp_out=.
-	em++ -s STANDALONE_WASM=1 -s EMIT_EMSCRIPTEN_METADATA=1 -s EXPORTED_FUNCTIONS=['_malloc'] --std=c++17 -O3 -I${CPP_API} -I${CPP_API}/google/protobuf -Iextensions/common -I. -I/usr/local/include -I${ABSL} --js-library ${CPP_API}/proxy_wasm_intrinsics.js ${ABSL_CPP} $*.cc ${CPP_API}/proxy_wasm_intrinsics.pb.cc ${PROTO_SRCS} ${COMMON_SRCS} ${CPP_CONTEXT_LIB} ${CPP_API}/libprotobuf.a -o $*.wasm
+	em++ -s STANDALONE_WASM=1 -s EMIT_EMSCRIPTEN_METADATA=1 -s EXPORTED_FUNCTIONS=['_malloc'] --std=c++17 -O0 -g4 --profiling --profiling-funcs -I${CPP_API} -I${CPP_API}/google/protobuf -Iextensions/common -I. -I/usr/local/include -I${ABSL} --js-library ${CPP_API}/proxy_wasm_intrinsics.js ${ABSL_CPP} $*.cc ${CPP_API}/proxy_wasm_intrinsics.pb.cc ${PROTO_SRCS} ${COMMON_SRCS} ${CPP_CONTEXT_LIB} ${CPP_API}/libprotobuf.a -o $*.wasm
 	rm -f $*.wast
 	rm -f extensions/metadata_exchange/declare_property.pb.*
 	chown ${uid}.${gid} $^
diff --git a/extensions/stats/Makefile b/extensions/stats/Makefile
index 957c7031..1f720d45 100644
--- a/extensions/stats/Makefile
+++ b/extensions/stats/Makefile
@@ -9,6 +9,6 @@ COMMON_SRCS = extensions/common/context.cc extensions/common/util.cc extensions/
 all: plugin.wasm
 
 %.wasm %.wat: %.cc ${CPP_API}/proxy_wasm_intrinsics.h ${CPP_API}/proxy_wasm_enums.h ${CPP_API}/proxy_wasm_externs.h ${CPP_API}/proxy_wasm_api.h ${CPP_API}/proxy_wasm_intrinsics.js ${CPP_CONTEXT_LIB}
-	em++ -s STANDALONE_WASM=1 -s EMIT_EMSCRIPTEN_METADATA=1 -s EXPORTED_FUNCTIONS=['_malloc'] --std=c++17 -O3 -I${CPP_API} -Iextensions/common -I. -I/usr/local/include -I${ABSL} -I. --js-library ${CPP_API}/proxy_wasm_intrinsics.js $*.cc ${COMMON_SRCS} ${CPP_CONTEXT_LIB} ${ABSL_CPP} -o $*.wasm
+	em++ -s STANDALONE_WASM=1 -s EMIT_EMSCRIPTEN_METADATA=1 -s EXPORTED_FUNCTIONS=['_malloc'] --std=c++17 -O0 -g4 --profiling --profiling-funcs -I${CPP_API} -Iextensions/common -I. -I/usr/local/include -I${ABSL} -I. --js-library ${CPP_API}/proxy_wasm_intrinsics.js $*.cc ${COMMON_SRCS} ${CPP_CONTEXT_LIB} ${ABSL_CPP} -o $*.wasm
 	rm -f $*.wast
 	chown ${uid}.${gid} $^
diff --git a/scripts/release-binary.sh b/scripts/release-binary.sh
index efe9e221..9a470e4b 100755
--- a/scripts/release-binary.sh
+++ b/scripts/release-binary.sh
@@ -19,7 +19,7 @@
 set -ex
 
 # Use clang for the release builds.
-export PATH=/usr/lib/llvm-9/bin:$PATH
+export PATH=/usr/lib/llvm/bin:$PATH
 export CC=${CC:-clang}
 export CXX=${CXX:-clang++}
 

zz v8-flags-from-env--istio-envoy.diff

diff --git a/bazel/external/wee8.BUILD b/bazel/external/wee8.BUILD
index c6e64f43bc..c37d2fcf1e 100644
--- a/bazel/external/wee8.BUILD
+++ b/bazel/external/wee8.BUILD
@@ -8,7 +8,11 @@ cc_library(
         "libwee8.a",
     ],
     hdrs = [
+        "wee8/include/v8.h",
         "wee8/include/v8-version.h",
+        "wee8/include/v8-profiler.h",
+        "wee8/include/v8-internal.h",
+        "wee8/include/v8config.h",
         "wee8/third_party/wasm-api/wasm.hh",
     ],
     includes = [
diff --git a/bazel/external/wee8.genrule_cmd b/bazel/external/wee8.genrule_cmd
index f62997af1f..cb62e38d57 100644
--- a/bazel/external/wee8.genrule_cmd
+++ b/bazel/external/wee8.genrule_cmd
@@ -73,6 +73,8 @@ WEE8_BUILD_ARGS+=" v8_use_external_startup_data=false"
 # Disable read-only heap, since it's leaky and HEAPCHECK complains about it.
 # TODO(PiotrSikora): remove when fixed upstream.
 WEE8_BUILD_ARGS+=" v8_enable_shared_ro_heap=false"
+WEE8_BUILD_ARGS+=" enable_profiling = true"
+
 # Support Arm64
 if [[ `uname -m` == "aarch64" ]]; then
   WEE8_BUILD_ARGS+=" target_cpu=\"arm64\""
diff --git a/source/extensions/common/wasm/v8/v8.cc b/source/extensions/common/wasm/v8/v8.cc
index 22a590eee4..2dd478b16c 100644
--- a/source/extensions/common/wasm/v8/v8.cc
+++ b/source/extensions/common/wasm/v8/v8.cc
@@ -11,16 +11,27 @@
 
 #include "absl/container/flat_hash_map.h"
 #include "absl/strings/match.h"
+#include "v8.h"
 #include "v8-version.h"
 #include "wasm-api/wasm.hh"
 
+#include <iostream>
+
 namespace Envoy {
 namespace Extensions {
 namespace Common {
 namespace Wasm {
 namespace V8 {
 
+// 
+v8::Local<v8::String> ReadWasmLoadSourceMap(v8::Isolate*, const char* name) {
+  (void)name;
+  return v8::Local<v8::String>();
+}
+
 wasm::Engine* engine() {
+  v8::V8::SetFlagsFromString(getenv("V8OPTS"));
+  //v8::V8::SetWasmLoadSourceMapCallback(ReadWasmLoadSourceMap);
   static const auto engine = wasm::Engine::make();
   return engine.get();
 }