-
-
Save iris-garden/bfacd8246bb8b29cd67caea063273c16 to your computer and use it in GitHub Desktop.
hail log dataproc notebook slowdown
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| 2023-09-22 17:09:54.767 SparkContext: INFO: Running Spark version 3.3.0 | |
| 2023-09-22 17:09:54.830 ResourceUtils: INFO: ============================================================== | |
| 2023-09-22 17:09:54.831 ResourceUtils: INFO: No custom resources configured for spark.driver. | |
| 2023-09-22 17:09:54.831 ResourceUtils: INFO: ============================================================== | |
| 2023-09-22 17:09:54.832 SparkContext: INFO: Submitted application: Hail | |
| 2023-09-22 17:09:54.842 SparkContext: INFO: Spark configuration: | |
| spark.app.name=Hail | |
| spark.app.startTime=1695402594764 | |
| spark.app.submitTime=1695402594229 | |
| spark.checkpoint.compress=true | |
| spark.dataproc.sql.joinConditionReorder.enabled=true | |
| spark.dataproc.sql.local.rank.pushdown.enabled=true | |
| spark.dataproc.sql.optimizer.leftsemijoin.conversion.enabled=true | |
| spark.dataproc.sql.parquet.enableFooterCache=true | |
| spark.driver.extraClassPath=/opt/conda/miniconda3/lib/python3.10/site-packages/hail/backend/hail-all-spark.jar,/opt/conda/miniconda3/lib/python3.10/site-packages/sparkmonitor/listener.jar | |
| spark.driver.extraJavaOptions=-XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Xss4M | |
| spark.driver.maxResultSize=0 | |
| spark.driver.memory=41g | |
| spark.dynamicAllocation.enabled=true | |
| spark.dynamicAllocation.maxExecutors=10000 | |
| spark.dynamicAllocation.minExecutors=1 | |
| spark.eventLog.dir=gs://dataproc-temp-us-central1-842871226259-nmtwlbu8/715f1e4f-db9f-4756-81fd-3031c77879b3/spark-job-history | |
| spark.eventLog.enabled=true | |
| spark.executor.cores=4 | |
| spark.executor.extraClassPath=./hail-all-spark.jar | |
| spark.executor.extraJavaOptions=-XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Xss4M | |
| spark.executor.instances=2 | |
| spark.executor.memory=5837m | |
| spark.executor.memoryOverhead=8755m | |
| spark.executorEnv.HAIL_WORKER_OFF_HEAP_MEMORY_PER_CORE_MB=3648 | |
| spark.executorEnv.OPENBLAS_NUM_THREADS=1 | |
| spark.executorEnv.PYTHONHASHSEED=0 | |
| spark.extraListeners=sparkmonitor.listener.JupyterSparkMonitorListener | |
| spark.hadoop.hive.execution.engine=mr | |
| spark.hadoop.io.compression.codecs=org.apache.hadoop.io.compress.DefaultCodec,is.hail.io.compress.BGzipCodec,is.hail.io.compress.BGzipCodecTbi,org.apache.hadoop.io.compress.GzipCodec | |
| spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version=2 | |
| spark.hadoop.mapreduce.input.fileinputformat.split.minsize=0 | |
| spark.history.fs.logDirectory=gs://dataproc-temp-us-central1-842871226259-nmtwlbu8/715f1e4f-db9f-4756-81fd-3031c77879b3/spark-job-history | |
| spark.jars=file:/opt/conda/miniconda3/lib/python3.10/site-packages/hail/backend/hail-all-spark.jar,file:/opt/conda/miniconda3/lib/python3.10/site-packages/sparkmonitor/listener.jar | |
| spark.kryo.registrator=is.hail.kryo.HailKryoRegistrator | |
| spark.kryoserializer.buffer.max=1g | |
| spark.logConf=true | |
| spark.master=yarn | |
| spark.memory.storageFraction=0.2 | |
| spark.metrics.namespace=app_name:${spark.app.name}.app_id:${spark.app.id} | |
| spark.repl.local.jars=file:///opt/conda/miniconda3/lib/python3.10/site-packages/hail/backend/hail-all-spark.jar,file:///opt/conda/miniconda3/lib/python3.10/site-packages/sparkmonitor/listener.jar | |
| spark.rpc.message.maxSize=512 | |
| spark.scheduler.minRegisteredResourcesRatio=0.0 | |
| spark.scheduler.mode=FAIR | |
| spark.serializer=org.apache.spark.serializer.KryoSerializer | |
| spark.shuffle.service.enabled=true | |
| spark.speculation=true | |
| spark.sql.adaptive.enabled=true | |
| spark.sql.autoBroadcastJoinThreshold=43m | |
| spark.sql.catalogImplementation=hive | |
| spark.sql.cbo.enabled=true | |
| spark.sql.cbo.joinReorder.enabled=true | |
| spark.sql.parquet.enableNestedColumnVectorizedReader=true | |
| spark.submit.deployMode=client | |
| spark.submit.pyFiles= | |
| spark.task.maxFailures=20 | |
| spark.ui.port=0 | |
| spark.ui.showConsoleProgress=false | |
| spark.yarn.am.memory=640m | |
| spark.yarn.dist.jars=file:///opt/conda/miniconda3/lib/python3.10/site-packages/hail/backend/hail-all-spark.jar,file:///opt/conda/miniconda3/lib/python3.10/site-packages/sparkmonitor/listener.jar | |
| spark.yarn.historyServer.address=notebook-slowdown-repro-m:18080 | |
| spark.yarn.isPython=true | |
| spark.yarn.jars=local:/usr/lib/spark/jars/* | |
| spark.yarn.unmanagedAM.enabled=true | |
| 2023-09-22 17:09:54.971 ResourceProfile: INFO: Default ResourceProfile created, executor resources: Map(memoryOverhead -> name: memoryOverhead, amount: 8755, script: , vendor: , cores -> name: cores, amount: 4, script: , vendor: , memory -> name: memory, amount: 5837, script: , vendor: , offHeap -> name: offHeap, amount: 0, script: , vendor: ), task resources: Map(cpus -> name: cpus, amount: 1.0) | |
| 2023-09-22 17:09:54.989 ResourceProfile: INFO: Limiting resource is cpus at 4 tasks per executor | |
| 2023-09-22 17:09:54.992 ResourceProfileManager: INFO: Added ResourceProfile id: 0 | |
| 2023-09-22 17:09:55.082 SecurityManager: INFO: Changing view acls to: root | |
| 2023-09-22 17:09:55.083 SecurityManager: INFO: Changing modify acls to: root | |
| 2023-09-22 17:09:55.083 SecurityManager: INFO: Changing view acls groups to: | |
| 2023-09-22 17:09:55.083 SecurityManager: INFO: Changing modify acls groups to: | |
| 2023-09-22 17:09:55.084 SecurityManager: INFO: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(root); groups with view permissions: Set(); users with modify permissions: Set(root); groups with modify permissions: Set() | |
| 2023-09-22 17:09:55.493 Utils: INFO: Successfully started service 'sparkDriver' on port 42559. | |
| 2023-09-22 17:09:55.598 SparkEnv: INFO: Registering MapOutputTracker | |
| 2023-09-22 17:09:55.671 SparkEnv: INFO: Registering BlockManagerMaster | |
| 2023-09-22 17:09:55.691 BlockManagerMasterEndpoint: INFO: Using org.apache.spark.storage.DefaultTopologyMapper for getting topology information | |
| 2023-09-22 17:09:55.692 BlockManagerMasterEndpoint: INFO: BlockManagerMasterEndpoint up | |
| 2023-09-22 17:09:55.736 SparkEnv: INFO: Registering BlockManagerMasterHeartbeat | |
| 2023-09-22 17:09:55.791 DiskBlockManager: INFO: Created local directory at /hadoop/spark/tmp/blockmgr-061b1186-6c32-4d6f-a1ea-c6176857ffb4 | |
| 2023-09-22 17:09:55.824 MemoryStore: INFO: MemoryStore started with capacity 24.4 GiB | |
| 2023-09-22 17:09:55.882 SparkEnv: INFO: Registering OutputCommitCoordinator | |
| 2023-09-22 17:09:55.953 log: INFO: Logging initialized @4827ms to org.sparkproject.jetty.util.log.Slf4jLog | |
| 2023-09-22 17:09:56.099 Server: INFO: jetty-9.4.43.v20210629; built: 2021-06-30T11:07:22.254Z; git: 526006ecfa3af7f1a27ef3a288e2bef7ea9dd7e8; jvm 11.0.17+8 | |
| 2023-09-22 17:09:56.126 Server: INFO: Started @5001ms | |
| 2023-09-22 17:09:56.168 AbstractConnector: INFO: Started ServerConnector@4c6e3b0d{HTTP/1.1, (http/1.1)}{0.0.0.0:43055} | |
| 2023-09-22 17:09:56.169 Utils: INFO: Successfully started service 'SparkUI' on port 43055. | |
| 2023-09-22 17:09:56.202 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@beecb72{/,null,AVAILABLE,@Spark} | |
| 2023-09-22 17:09:56.226 SparkContext: INFO: Added JAR file:/opt/conda/miniconda3/lib/python3.10/site-packages/hail/backend/hail-all-spark.jar at spark://notebook-slowdown-repro-m.c.broad-ctsa.internal:42559/jars/hail-all-spark.jar with timestamp 1695402594764 | |
| 2023-09-22 17:09:56.227 SparkContext: INFO: Added JAR file:/opt/conda/miniconda3/lib/python3.10/site-packages/sparkmonitor/listener.jar at spark://notebook-slowdown-repro-m.c.broad-ctsa.internal:42559/jars/listener.jar with timestamp 1695402594764 | |
| 2023-09-22 17:09:56.331 FairSchedulableBuilder: INFO: Creating Fair Scheduler pools from default file: fairscheduler.xml | |
| 2023-09-22 17:09:56.419 FairSchedulableBuilder: INFO: Created pool: default, schedulingMode: FAIR, minShare: 0, weight: 1 | |
| 2023-09-22 17:09:56.476 Utils: INFO: Using initial executors = 2, max of spark.dynamicAllocation.initialExecutors, spark.dynamicAllocation.minExecutors and spark.executor.instances | |
| 2023-09-22 17:09:56.946 DefaultNoHARMFailoverProxyProvider: INFO: Connecting to ResourceManager at notebook-slowdown-repro-m.c.broad-ctsa.internal./10.240.0.4:8032 | |
| 2023-09-22 17:09:57.238 AHSProxy: INFO: Connecting to Application History server at notebook-slowdown-repro-m.c.broad-ctsa.internal./10.240.0.4:10200 | |
| 2023-09-22 17:09:58.086 ResourceUtils: INFO: Unable to find 'resource-types.xml'. | |
| 2023-09-22 17:09:58.101 Client: INFO: Verifying our application has not requested more than the maximum memory capability of the cluster (14592 MB per container) | |
| 2023-09-22 17:09:58.102 Client: INFO: Will allocate AM container, with 1024 MB memory including 384 MB overhead | |
| 2023-09-22 17:09:58.103 Client: INFO: Setting up container launch context for our AM | |
| 2023-09-22 17:09:58.106 Client: INFO: Setting up the launch environment for our AM container | |
| 2023-09-22 17:09:58.113 Client: INFO: Preparing resources for our AM container | |
| 2023-09-22 17:09:58.168 Client: INFO: Uploading resource file:/opt/conda/miniconda3/lib/python3.10/site-packages/hail/backend/hail-all-spark.jar -> hdfs://notebook-slowdown-repro-m/user/root/.sparkStaging/application_1695402030462_0001/hail-all-spark.jar | |
| 2023-09-22 17:09:59.570 Client: INFO: Uploading resource file:/opt/conda/miniconda3/lib/python3.10/site-packages/sparkmonitor/listener.jar -> hdfs://notebook-slowdown-repro-m/user/root/.sparkStaging/application_1695402030462_0001/listener.jar | |
| 2023-09-22 17:09:59.601 Client: INFO: Uploading resource file:/usr/lib/spark/python/lib/pyspark.zip -> hdfs://notebook-slowdown-repro-m/user/root/.sparkStaging/application_1695402030462_0001/pyspark.zip | |
| 2023-09-22 17:09:59.644 Client: INFO: Uploading resource file:/usr/lib/spark/python/lib/py4j-0.10.9.5-src.zip -> hdfs://notebook-slowdown-repro-m/user/root/.sparkStaging/application_1695402030462_0001/py4j-0.10.9.5-src.zip | |
| 2023-09-22 17:09:59.872 Client: INFO: Uploading resource file:/hadoop/spark/tmp/spark-7a2aa5f3-c3cf-42ae-b58e-b67bd1c5c487/__spark_conf__1662560229723861471.zip -> hdfs://notebook-slowdown-repro-m/user/root/.sparkStaging/application_1695402030462_0001/__spark_conf__.zip | |
| 2023-09-22 17:09:59.937 SecurityManager: INFO: Changing view acls to: root | |
| 2023-09-22 17:09:59.937 SecurityManager: INFO: Changing modify acls to: root | |
| 2023-09-22 17:09:59.937 SecurityManager: INFO: Changing view acls groups to: | |
| 2023-09-22 17:09:59.937 SecurityManager: INFO: Changing modify acls groups to: | |
| 2023-09-22 17:09:59.938 SecurityManager: INFO: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(root); groups with view permissions: Set(); users with modify permissions: Set(root); groups with modify permissions: Set() | |
| 2023-09-22 17:09:59.962 Client: INFO: Submitting application application_1695402030462_0001 to ResourceManager | |
| 2023-09-22 17:10:00.328 YarnClientImpl: INFO: Submitted application application_1695402030462_0001 | |
| 2023-09-22 17:10:01.334 Client: INFO: Application report for application_1695402030462_0001 (state: ACCEPTED) | |
| 2023-09-22 17:10:01.337 Client: INFO: | |
| client token: N/A | |
| diagnostics: AM container is launched, waiting for AM container to Register with RM | |
| ApplicationMaster host: N/A | |
| ApplicationMaster RPC port: -1 | |
| queue: default | |
| start time: 1695402600027 | |
| final status: UNDEFINED | |
| tracking URL: http://notebook-slowdown-repro-m.c.broad-ctsa.internal.:8088/proxy/application_1695402030462_0001/ | |
| user: root | |
| 2023-09-22 17:10:01.345 SecurityManager: INFO: Changing view acls to: root | |
| 2023-09-22 17:10:01.345 SecurityManager: INFO: Changing modify acls to: root | |
| 2023-09-22 17:10:01.345 SecurityManager: INFO: Changing view acls groups to: | |
| 2023-09-22 17:10:01.345 SecurityManager: INFO: Changing modify acls groups to: | |
| 2023-09-22 17:10:01.345 SecurityManager: INFO: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(root); groups with view permissions: Set(); users with modify permissions: Set(root); groups with modify permissions: Set() | |
| 2023-09-22 17:10:01.392 DefaultNoHARMFailoverProxyProvider: INFO: Connecting to ResourceManager at notebook-slowdown-repro-m.c.broad-ctsa.internal./10.240.0.4:8030 | |
| 2023-09-22 17:10:01.420 YarnRMClient: INFO: Registering the ApplicationMaster | |
| 2023-09-22 17:10:01.547 YarnClientSchedulerBackend: INFO: Add WebUI Filter. org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter, Map(PROXY_HOSTS -> notebook-slowdown-repro-m.c.broad-ctsa.internal., PROXY_URI_BASES -> http://notebook-slowdown-repro-m.c.broad-ctsa.internal.:8088/proxy/application_1695402030462_0001), /proxy/application_1695402030462_0001 | |
| 2023-09-22 17:10:01.563 ApplicationMaster: INFO: Preparing Local resources | |
| 2023-09-22 17:10:01.642 ApplicationMaster: INFO: | |
| =============================================================================== | |
| Default YARN executor launch context: | |
| env: | |
| SPARK_MONITOR_UI -> http://localhost:8088/proxy/%APP_ID% | |
| SPARK_WORKER_WEBUI_PORT -> 18081 | |
| SPARK_ENV_LOADED -> 1 | |
| CLASSPATH -> ./hail-all-spark.jar<CPS>{{PWD}}<CPS>{{PWD}}/__spark_conf__<CPS>{{PWD}}/__spark_libs__/*<CPS>/usr/lib/spark/jars/*<CPS>:/etc/hive/conf:/usr/local/share/google/dataproc/lib/*:/usr/share/java/mysql.jar<CPS>{{PWD}}/__spark_conf__/__hadoop_conf__ | |
| SPARK_LOG_DIR -> /var/log/spark | |
| SPARK_LOCAL_DIRS -> /hadoop/spark/tmp | |
| SPARK_DIST_CLASSPATH -> :/etc/hive/conf:/usr/local/share/google/dataproc/lib/*:/usr/share/java/mysql.jar | |
| SPARK_USER -> root | |
| SPARK_SUBMIT_OPTS -> -Dscala.usejavacp=true | |
| SPARKMONITOR_KERNEL_PORT -> 55989 | |
| SPARK_CONF_DIR -> /usr/lib/spark//conf | |
| HAIL_WORKER_OFF_HEAP_MEMORY_PER_CORE_MB -> 3648 | |
| PYTHONHASHSEED -> 0 | |
| SPARK_HOME -> /usr/lib/spark/ | |
| PYTHONPATH -> /usr/lib/spark/python/lib/pyspark.zip:/usr/lib/spark/python/lib/py4j-0.10.9.5-src.zip<CPS>{{PWD}}/pyspark.zip<CPS>{{PWD}}/py4j-0.10.9.5-src.zip | |
| SPARK_MASTER_PORT -> 7077 | |
| OPENBLAS_NUM_THREADS -> 1 | |
| SPARK_WORKER_DIR -> /hadoop/spark/work | |
| SPARK_WORKER_PORT -> 7078 | |
| SPARK_DAEMON_MEMORY -> 4000m | |
| SPARK_MASTER_WEBUI_PORT -> 18080 | |
| SPARK_LIBRARY_PATH -> :/usr/lib/hadoop/lib/native | |
| SPARK_SCALA_VERSION -> 2.12 | |
| command: | |
| {{JAVA_HOME}}/bin/java \ | |
| -server \ | |
| -Xmx5837m \ | |
| '-XX:+IgnoreUnrecognizedVMOptions' \ | |
| '--add-opens=java.base/java.lang=ALL-UNNAMED' \ | |
| '--add-opens=java.base/java.lang.invoke=ALL-UNNAMED' \ | |
| '--add-opens=java.base/java.lang.reflect=ALL-UNNAMED' \ | |
| '--add-opens=java.base/java.io=ALL-UNNAMED' \ | |
| '--add-opens=java.base/java.net=ALL-UNNAMED' \ | |
| '--add-opens=java.base/java.nio=ALL-UNNAMED' \ | |
| '--add-opens=java.base/java.util=ALL-UNNAMED' \ | |
| '--add-opens=java.base/java.util.concurrent=ALL-UNNAMED' \ | |
| '--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED' \ | |
| '--add-opens=java.base/sun.nio.ch=ALL-UNNAMED' \ | |
| '--add-opens=java.base/sun.nio.cs=ALL-UNNAMED' \ | |
| '--add-opens=java.base/sun.security.action=ALL-UNNAMED' \ | |
| '--add-opens=java.base/sun.util.calendar=ALL-UNNAMED' \ | |
| '--add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED' \ | |
| '-Xss4M' \ | |
| -Djava.io.tmpdir={{PWD}}/tmp \ | |
| '-Dspark.driver.port=42559' \ | |
| '-Dspark.ui.port=0' \ | |
| '-Dspark.rpc.message.maxSize=512' \ | |
| -Dspark.yarn.app.container.log.dir=<LOG_DIR> \ | |
| -XX:OnOutOfMemoryError='kill %p' \ | |
| org.apache.spark.executor.YarnCoarseGrainedExecutorBackend \ | |
| --driver-url \ | |
| spark://CoarseGrainedScheduler@notebook-slowdown-repro-m.c.broad-ctsa.internal:42559 \ | |
| --executor-id \ | |
| <executorId> \ | |
| --hostname \ | |
| <hostname> \ | |
| --cores \ | |
| 4 \ | |
| --app-id \ | |
| application_1695402030462_0001 \ | |
| --resourceProfileId \ | |
| 0 \ | |
| 1><LOG_DIR>/stdout \ | |
| 2><LOG_DIR>/stderr | |
| resources: | |
| __spark_conf__ -> resource { scheme: "hdfs" host: "notebook-slowdown-repro-m" port: -1 file: "/user/root/.sparkStaging/application_1695402030462_0001/__spark_conf__.zip" } size: 290520 timestamp: 1695402599910 type: ARCHIVE visibility: PRIVATE | |
| listener.jar -> resource { scheme: "hdfs" host: "notebook-slowdown-repro-m" port: -1 file: "/user/root/.sparkStaging/application_1695402030462_0001/listener.jar" } size: 37657 timestamp: 1695402599592 type: FILE visibility: PRIVATE | |
| pyspark.zip -> resource { scheme: "hdfs" host: "notebook-slowdown-repro-m" port: -1 file: "/user/root/.sparkStaging/application_1695402030462_0001/pyspark.zip" } size: 1529089 timestamp: 1695402599635 type: FILE visibility: PRIVATE | |
| py4j-0.10.9.5-src.zip -> resource { scheme: "hdfs" host: "notebook-slowdown-repro-m" port: -1 file: "/user/root/.sparkStaging/application_1695402030462_0001/py4j-0.10.9.5-src.zip" } size: 42404 timestamp: 1695402599663 type: FILE visibility: PRIVATE | |
| hail-all-spark.jar -> resource { scheme: "hdfs" host: "notebook-slowdown-repro-m" port: -1 file: "/user/root/.sparkStaging/application_1695402030462_0001/hail-all-spark.jar" } size: 186721849 timestamp: 1695402599526 type: FILE visibility: PRIVATE | |
| =============================================================================== | |
| 2023-09-22 17:10:01.679 Utils: INFO: Using initial executors = 2, max of spark.dynamicAllocation.initialExecutors, spark.dynamicAllocation.minExecutors and spark.executor.instances | |
| 2023-09-22 17:10:01.680 YarnAllocator: INFO: Resource profile 0 doesn't exist, adding it | |
| 2023-09-22 17:10:01.689 YarnSchedulerBackend$YarnSchedulerEndpoint: INFO: ApplicationMaster registered as NettyRpcEndpointRef(spark://YarnAM@notebook-slowdown-repro-m.c.broad-ctsa.internal:42559) | |
| 2023-09-22 17:10:01.699 YarnAllocator: INFO: Will request 2 executor container(s) for ResourceProfile Id: 0, each with 4 core(s) and 14592 MB memory. with custom resources: <memory:14592, vCores:4> | |
| 2023-09-22 17:10:01.705 YarnContainerInfoHelper: INFO: Error while building executor logs - executor logs will not be available | |
| java.lang.NullPointerException: null | |
| at org.apache.hadoop.yarn.api.records.ContainerId.fromString(ContainerId.java:190) ~[hadoop-client-api-3.3.3.jar:?] | |
| at org.apache.hadoop.yarn.util.ConverterUtils.toContainerId(ConverterUtils.java:140) ~[hadoop-client-api-3.3.3.jar:?] | |
| at org.apache.spark.deploy.yarn.YarnSparkHadoopUtil$.getContainerId(YarnSparkHadoopUtil.scala:252) ~[spark-yarn_2.12-3.3.0.jar:3.3.0] | |
| at org.apache.spark.util.YarnContainerInfoHelper$.getContainerId(YarnContainerInfoHelper.scala:90) ~[spark-yarn_2.12-3.3.0.jar:3.3.0] | |
| at org.apache.spark.util.YarnContainerInfoHelper$.getLogUrls(YarnContainerInfoHelper.scala:47) ~[spark-yarn_2.12-3.3.0.jar:3.3.0] | |
| at org.apache.spark.deploy.yarn.ApplicationMaster.org$apache$spark$deploy$yarn$ApplicationMaster$$extractLogUrls(ApplicationMaster.scala:71) ~[spark-yarn_2.12-3.3.0.jar:3.3.0] | |
| at org.apache.spark.deploy.yarn.ApplicationMaster$AMEndpoint.onStart(ApplicationMaster.scala:796) ~[spark-yarn_2.12-3.3.0.jar:3.3.0] | |
| at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:120) ~[spark-core_2.12-3.3.0.jar:3.3.0] | |
| at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:213) ~[spark-core_2.12-3.3.0.jar:3.3.0] | |
| at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:100) ~[spark-core_2.12-3.3.0.jar:3.3.0] | |
| at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:75) ~[spark-core_2.12-3.3.0.jar:3.3.0] | |
| at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:41) ~[spark-core_2.12-3.3.0.jar:3.3.0] | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?] | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?] | |
| at java.lang.Thread.run(Thread.java:829) ~[?:?] | |
| 2023-09-22 17:10:01.719 YarnAllocator: INFO: Submitted 2 unlocalized container requests. | |
| 2023-09-22 17:10:02.036 StatsdSink: INFO: StatsdSink started with prefix: 'spark.applicationMaster' | |
| 2023-09-22 17:10:02.037 ApplicationMaster: INFO: Started progress reporter thread with (heartbeat : 3000, initial allocation : 200) intervals | |
| 2023-09-22 17:10:02.099 YarnAllocator: INFO: Launching container container_1695402030462_0001_01_000001 on host notebook-slowdown-repro-w-1.c.broad-ctsa.internal for executor with ID 1 for ResourceProfile Id 0 | |
| 2023-09-22 17:10:02.102 YarnAllocator: INFO: Received 1 containers from YARN, launching executors on 1 of them. | |
| 2023-09-22 17:10:02.116 ExecutorRunnable: INFO: Initializing service data for shuffle service using name 'spark_shuffle' | |
| 2023-09-22 17:10:02.373 Client: INFO: Application report for application_1695402030462_0001 (state: RUNNING) | |
| 2023-09-22 17:10:02.374 Client: INFO: | |
| client token: N/A | |
| diagnostics: N/A | |
| ApplicationMaster host: 10.240.0.4 | |
| ApplicationMaster RPC port: -1 | |
| queue: default | |
| start time: 1695402600027 | |
| final status: UNDEFINED | |
| tracking URL: http://notebook-slowdown-repro-m.c.broad-ctsa.internal.:8088/proxy/application_1695402030462_0001/ | |
| user: root | |
| 2023-09-22 17:10:02.377 YarnClientSchedulerBackend: INFO: Application application_1695402030462_0001 has started running. | |
| 2023-09-22 17:10:02.379 YarnScheduler: INFO: Starting speculative execution thread | |
| 2023-09-22 17:10:02.390 Utils: INFO: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 38207. | |
| 2023-09-22 17:10:02.390 NettyBlockTransferService: INFO: Server created on notebook-slowdown-repro-m.c.broad-ctsa.internal:38207 | |
| 2023-09-22 17:10:02.392 BlockManager: INFO: Using org.apache.spark.storage.RandomBlockReplicationPolicy for block replication policy | |
| 2023-09-22 17:10:02.408 BlockManagerMaster: INFO: Registering BlockManager BlockManagerId(driver, notebook-slowdown-repro-m.c.broad-ctsa.internal, 38207, None) | |
| 2023-09-22 17:10:02.412 BlockManagerMasterEndpoint: INFO: Registering block manager notebook-slowdown-repro-m.c.broad-ctsa.internal:38207 with 24.4 GiB RAM, BlockManagerId(driver, notebook-slowdown-repro-m.c.broad-ctsa.internal, 38207, None) | |
| 2023-09-22 17:10:02.416 BlockManagerMaster: INFO: Registered BlockManager BlockManagerId(driver, notebook-slowdown-repro-m.c.broad-ctsa.internal, 38207, None) | |
| 2023-09-22 17:10:02.417 BlockManager: INFO: external shuffle service port = 7337 | |
| 2023-09-22 17:10:02.418 BlockManager: INFO: Initialized BlockManager: BlockManagerId(driver, notebook-slowdown-repro-m.c.broad-ctsa.internal, 38207, None) | |
| 2023-09-22 17:10:02.435 StatsdSink: INFO: StatsdSink started with prefix: 'spark.driver' | |
| 2023-09-22 17:10:02.727 YarnAllocator: INFO: Launching container container_1695402030462_0001_01_000002 on host notebook-slowdown-repro-w-0.c.broad-ctsa.internal for executor with ID 2 for ResourceProfile Id 0 | |
| 2023-09-22 17:10:02.727 YarnAllocator: INFO: Received 1 containers from YARN, launching executors on 1 of them. | |
| 2023-09-22 17:10:02.733 ExecutorRunnable: INFO: Initializing service data for shuffle service using name 'spark_shuffle' | |
| 2023-09-22 17:10:03.574 GoogleCloudStorageImpl: INFO: Ignoring exception of type GoogleJsonResponseException; verified object already exists with desired state. | |
| 2023-09-22 17:10:03.688 SingleEventLogFileWriter: INFO: Logging events to gs://dataproc-temp-us-central1-842871226259-nmtwlbu8/715f1e4f-db9f-4756-81fd-3031c77879b3/spark-job-history/application_1695402030462_0001.inprogress | |
| 2023-09-22 17:10:03.979 Utils: INFO: Using initial executors = 2, max of spark.dynamicAllocation.initialExecutors, spark.dynamicAllocation.minExecutors and spark.executor.instances | |
| 2023-09-22 17:10:03.996 YarnAllocator: INFO: Resource profile 0 doesn't exist, adding it | |
| 2023-09-22 17:10:04.016 SparkContext: INFO: Registered listener sparkmonitor.listener.JupyterSparkMonitorListener | |
| 2023-09-22 17:10:04.039 ContextHandler: INFO: Stopped o.s.j.s.ServletContextHandler@beecb72{/,null,STOPPED,@Spark} | |
| 2023-09-22 17:10:04.042 ServerInfo: INFO: Adding filter to /jobs: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter | |
| 2023-09-22 17:10:04.049 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@7c1d0a93{/jobs,null,AVAILABLE,@Spark} | |
| 2023-09-22 17:10:04.050 ServerInfo: INFO: Adding filter to /jobs/json: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter | |
| 2023-09-22 17:10:04.052 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@50072d14{/jobs/json,null,AVAILABLE,@Spark} | |
| 2023-09-22 17:10:04.053 ServerInfo: INFO: Adding filter to /jobs/job: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter | |
| 2023-09-22 17:10:04.055 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@15da6457{/jobs/job,null,AVAILABLE,@Spark} | |
| 2023-09-22 17:10:04.056 ServerInfo: INFO: Adding filter to /jobs/job/json: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter | |
| 2023-09-22 17:10:04.063 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@db19a{/jobs/job/json,null,AVAILABLE,@Spark} | |
| 2023-09-22 17:10:04.063 ServerInfo: INFO: Adding filter to /stages: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter | |
| 2023-09-22 17:10:04.065 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@48d9f57a{/stages,null,AVAILABLE,@Spark} | |
| 2023-09-22 17:10:04.065 ServerInfo: INFO: Adding filter to /stages/json: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter | |
| 2023-09-22 17:10:04.067 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@7e89235d{/stages/json,null,AVAILABLE,@Spark} | |
| 2023-09-22 17:10:04.067 ServerInfo: INFO: Adding filter to /stages/stage: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter | |
| 2023-09-22 17:10:04.071 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@918d196{/stages/stage,null,AVAILABLE,@Spark} | |
| 2023-09-22 17:10:04.071 ServerInfo: INFO: Adding filter to /stages/stage/json: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter | |
| 2023-09-22 17:10:04.073 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@41ff64f6{/stages/stage/json,null,AVAILABLE,@Spark} | |
| 2023-09-22 17:10:04.075 ServerInfo: INFO: Adding filter to /stages/pool: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter | |
| 2023-09-22 17:10:04.076 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@738a13f2{/stages/pool,null,AVAILABLE,@Spark} | |
| 2023-09-22 17:10:04.077 ServerInfo: INFO: Adding filter to /stages/pool/json: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter | |
| 2023-09-22 17:10:04.079 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@2661934d{/stages/pool/json,null,AVAILABLE,@Spark} | |
| 2023-09-22 17:10:04.080 ServerInfo: INFO: Adding filter to /storage: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter | |
| 2023-09-22 17:10:04.081 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@aeaf561{/storage,null,AVAILABLE,@Spark} | |
| 2023-09-22 17:10:04.082 ServerInfo: INFO: Adding filter to /storage/json: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter | |
| 2023-09-22 17:10:04.083 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@29a6c2f9{/storage/json,null,AVAILABLE,@Spark} | |
| 2023-09-22 17:10:04.083 ServerInfo: INFO: Adding filter to /storage/rdd: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter | |
| 2023-09-22 17:10:04.084 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@672717a6{/storage/rdd,null,AVAILABLE,@Spark} | |
| 2023-09-22 17:10:04.085 ServerInfo: INFO: Adding filter to /storage/rdd/json: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter | |
| 2023-09-22 17:10:04.086 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@6ba041d{/storage/rdd/json,null,AVAILABLE,@Spark} | |
| 2023-09-22 17:10:04.086 ServerInfo: INFO: Adding filter to /environment: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter | |
| 2023-09-22 17:10:04.087 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@464209cb{/environment,null,AVAILABLE,@Spark} | |
| 2023-09-22 17:10:04.087 ServerInfo: INFO: Adding filter to /environment/json: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter | |
| 2023-09-22 17:10:04.088 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@6cb63859{/environment/json,null,AVAILABLE,@Spark} | |
| 2023-09-22 17:10:04.089 ServerInfo: INFO: Adding filter to /executors: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter | |
| 2023-09-22 17:10:04.090 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@14e65e8a{/executors,null,AVAILABLE,@Spark} | |
| 2023-09-22 17:10:04.090 ServerInfo: INFO: Adding filter to /executors/json: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter | |
| 2023-09-22 17:10:04.091 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@38e68ac8{/executors/json,null,AVAILABLE,@Spark} | |
| 2023-09-22 17:10:04.092 ServerInfo: INFO: Adding filter to /executors/threadDump: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter | |
| 2023-09-22 17:10:04.094 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@78fe5a0c{/executors/threadDump,null,AVAILABLE,@Spark} | |
| 2023-09-22 17:10:04.094 ServerInfo: INFO: Adding filter to /executors/threadDump/json: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter | |
| 2023-09-22 17:10:04.096 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@2e6f589f{/executors/threadDump/json,null,AVAILABLE,@Spark} | |
| 2023-09-22 17:10:04.096 ServerInfo: INFO: Adding filter to /static: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter | |
| 2023-09-22 17:10:04.139 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@3c3fa609{/static,null,AVAILABLE,@Spark} | |
| 2023-09-22 17:10:04.139 ServerInfo: INFO: Adding filter to /: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter | |
| 2023-09-22 17:10:04.141 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@664e15a4{/,null,AVAILABLE,@Spark} | |
| 2023-09-22 17:10:04.142 ServerInfo: INFO: Adding filter to /api: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter | |
| 2023-09-22 17:10:04.144 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@33bc35be{/api,null,AVAILABLE,@Spark} | |
| 2023-09-22 17:10:04.144 ServerInfo: INFO: Adding filter to /jobs/job/kill: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter | |
| 2023-09-22 17:10:04.154 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@172b289e{/jobs/job/kill,null,AVAILABLE,@Spark} | |
| 2023-09-22 17:10:04.154 ServerInfo: INFO: Adding filter to /stages/stage/kill: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter | |
| 2023-09-22 17:10:04.156 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@424737be{/stages/stage/kill,null,AVAILABLE,@Spark} | |
| 2023-09-22 17:10:04.162 ServerInfo: INFO: Adding filter to /metrics/json: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter | |
| 2023-09-22 17:10:04.164 ContextHandler: INFO: Started o.s.j.s.ServletContextHandler@28ad0484{/metrics/json,null,AVAILABLE,@Spark} | |
| 2023-09-22 17:10:04.165 YarnClientSchedulerBackend: INFO: SchedulerBackend is ready for scheduling beginning after reached minRegisteredResourcesRatio: 0.0 | |
| 2023-09-22 17:10:04.178 Hail: INFO: SparkUI: http://notebook-slowdown-repro-m.c.broad-ctsa.internal:43055 | |
| 2023-09-22 17:10:04.571 Hail: INFO: Running Hail version 0.2.124-ee7fef6fc40d | |
| 2023-09-22 17:10:05.231 SparkContext: WARN: Using an existing SparkContext; some configuration may not take effect. | |
| 2023-09-22 17:10:07.740 : INFO: RegionPool: initialized for thread 17: Thread-3 | |
| 2023-09-22 17:10:08.334 : INFO: RegionPool: initialized for thread 17: Thread-3 | |
| 2023-09-22 17:10:08.336 : INFO: TaskReport: stage=0, partition=0, attempt=0, peakBytes=0, peakBytesReadable=0.00 B, chunks requested=0, cache hits=0 | |
| 2023-09-22 17:10:08.337 : INFO: RegionPool: FREE: 0 allocated (0 blocks / 0 chunks), regions.size = 0, 0 current java objects, thread 17: Thread-3 | |
| 2023-09-22 17:10:08.338 : INFO: RegionPool: FREE: 64.0K allocated (64.0K blocks / 0 chunks), regions.size = 1, 0 current java objects, thread 17: Thread-3 | |
| 2023-09-22 17:10:08.339 : INFO: timing SparkBackend.parse_value_ir total 605.670ms self 605.670ms children 0.000ms %children 0.00% | |
| 2023-09-22 17:10:08.340 : INFO: RegionPool: initialized for thread 17: Thread-3 | |
| 2023-09-22 17:10:08.350 : INFO: starting execution of query hail_query_1 of initial size 22 | |
| 2023-09-22 17:10:08.456 : WARN: Failed to compute SemanticHash: SemanticHash unknown: is.hail.expr.ir.TableHead | |
| 2023-09-22 17:10:08.457 : INFO: IR Semantic Hash: None | |
| 2023-09-22 17:10:08.653 : INFO: initial IR: IR size 22: | |
| %1 = RNGStateLiteral | |
| !ht = TableRange [10, 2] | |
| !ht2 = TableOrderBy(!ht) [(Aidx)] | |
| !ht3 = TableMapRows(!ht2) { (%g, %row) => | |
| !2 = SelectFields(%row) [(idx)] | |
| !3 = SelectFields(!2) [()] | |
| !4 = GetField(%row) [idx] | |
| InsertFields !3 (idx: !4) | |
| } | |
| !ht4 = TableMapRows(!ht3) { (%g2, %row2) => | |
| !5 = SelectFields(%row2) [(idx)] | |
| !6 = SelectFields(!5) [()] | |
| !7 = GetField(%row2) [idx] | |
| !8 = ApplySpecial(!7) [1, showStr, (), String] | |
| InsertFields !6 (idx: !8) | |
| } | |
| !ht5 = TableHead(!ht4) [11] | |
| !9 = TableCollect(!ht5) | |
| GetField(!9) [rows] | |
| 2023-09-22 17:10:08.811 : INFO: after optimize: relationalLowerer, initial IR: IR size 12: | |
| !ht = TableRange [10, 2] | |
| !ht2 = TableOrderBy(!ht) [(Aidx)] | |
| !ht3 = TableHead(!ht2) [11] | |
| !ht4 = TableMapRows(!ht3) { (%g, %row) => | |
| !1 = SelectFields(%row) [()] | |
| !2 = GetField(%row) [idx] | |
| !3 = ApplySpecial(!2) [1, showStr, (), String] | |
| InsertFields !1 (idx: !3) | |
| } | |
| !4 = TableCollect(!ht4) | |
| GetField(!4) [rows] | |
| 2023-09-22 17:10:08.828 : INFO: after LowerMatrixToTable: IR size 12: | |
| !ht = TableRange [10, 2] | |
| !ht2 = TableOrderBy(!ht) [(Aidx)] | |
| !ht3 = TableHead(!ht2) [11] | |
| !ht4 = TableMapRows(!ht3) { (%g, %row) => | |
| !1 = SelectFields(%row) [()] | |
| !2 = GetField(%row) [idx] | |
| !3 = ApplySpecial(!2) [1, showStr, (), String] | |
| InsertFields !1 (idx: !3) | |
| } | |
| !4 = TableCollect(!ht4) | |
| GetField(!4) [rows] | |
| 2023-09-22 17:10:08.839 : INFO: after optimize: relationalLowerer, after LowerMatrixToTable: IR size 12: | |
| !ht = TableRange [10, 2] | |
| !ht2 = TableOrderBy(!ht) [(Aidx)] | |
| !ht3 = TableHead(!ht2) [11] | |
| !ht4 = TableMapRows(!ht3) { (%g, %row) => | |
| !1 = SelectFields(%row) [()] | |
| !2 = GetField(%row) [idx] | |
| !3 = ApplySpecial(!2) [1, showStr, (), String] | |
| InsertFields !1 (idx: !3) | |
| } | |
| !4 = TableCollect(!ht4) | |
| GetField(!4) [rows] | |
| 2023-09-22 17:10:08.845 : INFO: after LiftRelationalValuesToRelationalLets: IR size 14: | |
| !ht = TableRange [10, 2] | |
| !ht2 = TableOrderBy(!ht) [(Aidx)] | |
| !ht3 = TableHead(!ht2) [11] | |
| !ht4 = TableMapRows(!ht3) { (%g, %row) => | |
| !1 = SelectFields(%row) [()] | |
| !2 = GetField(%row) [idx] | |
| !3 = ApplySpecial(!2) [1, showStr, (), String] | |
| InsertFields !1 (idx: !3) | |
| } | |
| %4 = TableCollect(!ht4) | |
| GetField(%4) [rows] | |
| 2023-09-22 17:10:08.849 : INFO: initial IR: IR size 11: | |
| !ht = TableRange [10, 2] | |
| !ht2 = TableOrderBy(!ht) [(Aidx)] | |
| !ht3 = TableHead(!ht2) [11] | |
| !ht4 = TableMapRows(!ht3) { (%g, %row) => | |
| !1 = SelectFields(%row) [()] | |
| !2 = GetField(%row) [idx] | |
| !3 = ApplySpecial(!2) [1, showStr, (), String] | |
| InsertFields !1 (idx: !3) | |
| } | |
| TableCollect(!ht4) | |
| 2023-09-22 17:10:08.853 : INFO: after LowerAndExecuteShuffles: IR size 11: | |
| !ht = TableRange [10, 2] | |
| !ht2 = TableOrderBy(!ht) [(Aidx)] | |
| !ht3 = TableHead(!ht2) [11] | |
| !ht4 = TableMapRows(!ht3) { (%g, %row) => | |
| !1 = SelectFields(%row) [()] | |
| !2 = GetField(%row) [idx] | |
| !3 = ApplySpecial(!2) [1, showStr, (), String] | |
| InsertFields !1 (idx: !3) | |
| } | |
| TableCollect(!ht4) | |
| 2023-09-22 17:10:08.862 : INFO: after optimize: relationalLowerer, after LowerAndExecuteShuffles: IR size 11: | |
| !ht = TableRange [10, 2] | |
| !ht2 = TableOrderBy(!ht) [(Aidx)] | |
| !ht3 = TableHead(!ht2) [11] | |
| !ht4 = TableMapRows(!ht3) { (%g, %row) => | |
| !1 = SelectFields(%row) [()] | |
| !2 = GetField(%row) [idx] | |
| !3 = ApplySpecial(!2) [1, showStr, (), String] | |
| InsertFields !1 (idx: !3) | |
| } | |
| TableCollect(!ht4) | |
| 2023-09-22 17:10:08.870 : INFO: LowerOrInterpretNonCompilable: whole stage code generation is a go! | |
| 2023-09-22 17:10:08.871 : INFO: lowering result: TableCollect | |
| 2023-09-22 17:10:09.102 : INFO: initial IR: IR size 148: | |
| %1 = MakeStruct() | |
| !2 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s = ToStream(!2) [False] | |
| %3 = ToArray(!s) | |
| !c4 = I32 [4] | |
| !c0 = I32 [0] | |
| %30 = TailLoop(!c4, !c0) { | |
| (%loopvar, %loopvar2, %loop) => | |
| %4 = MakeStruct() | |
| !s2 = ToStream(%3) [False] | |
| !s3 = StreamTake(!s2, %loopvar) | |
| !5 = MakeStruct(__iruid_3: %4) | |
| !6 = Str ["iteration="] | |
| !7 = Apply(%loopvar2) [-1, str, (), String] | |
| !8 = Apply(!6, !7) [-1, concat, (), String] | |
| !9 = Str [",nParts="] | |
| !10 = Apply(!8, !9) [-1, concat, (), String] | |
| !11 = Apply(%loopvar) [-1, str, (), String] | |
| !12 = Apply(!10, !11) [-1, concat, (), String] | |
| %18 = CollectDistributedArray(!s3, !5, !12) { | |
| (%ctx, %g) => | |
| %13 = GetField(%g) [__iruid_3] | |
| %14 = Ref [__iruid_10] | |
| %15 = Ref [__iruid_11] | |
| !16 = GetField(%15) [start] | |
| !17 = GetField(%15) [end] | |
| !c1 = I32 [1] | |
| !s4 = StreamRange(!16, !17, !c1) [-1, True] | |
| !s5 = StreamMap(!s4) { (%elt) => | |
| MakeStruct(idx: %elt) | |
| } | |
| !c11 = I32 [11] | |
| !s6 = StreamTake(!s5, !c11) | |
| StreamLen(!s6) | |
| } | |
| %19 = Ref [__iruid_13] | |
| !s7 = ToStream(%19) [False] | |
| !c0_2 = I32 [0] | |
| !20 = StreamFold(!s7, !c0_2) { (%accum, %elt2) => | |
| ApplyBinaryPrimOp(%accum, %elt2) [Add] | |
| } | |
| !21 = Cast(!20) [Int64] | |
| !22 = I64 [11] | |
| !23 = ApplyComparisonOp(!21, !22) [GTEQ] | |
| !24 = ArrayLen(%3) | |
| !25 = ArrayLen(%19) | |
| !26 = ApplyComparisonOp(!24, !25) [LTEQ] | |
| !27 = ApplySpecial(!23, !26) [-1, lor, | |
| (), Boolean] | |
| If !27 then { %19 } else { | |
| !c4_2 = I32 [4] | |
| !28 = ApplyBinaryPrimOp(%loopvar, !c4_2) [Multiply] | |
| !c1_2 = I32 [1] | |
| !29 = ApplyBinaryPrimOp(%loopvar2, !c1_2) [Add] | |
| Recur(!28, !29) [Array[Int32]] | |
| } | |
| } | |
| %31 = ArrayLen(%30) | |
| !c0_3 = I32 [0] | |
| !32 = ApplyComparisonOp(%31, !c0_3) [EQWithNA] | |
| %48 = If !32 then { | |
| !c0_4 = I32 [0] | |
| !33 = I64 [0] | |
| MakeTuple(!c0_4, !33) [(0 1)] | |
| } else { | |
| !c0_5 = I32 [0] | |
| !34 = I64 [11] | |
| TailLoop(!c0_5, !34) { | |
| (%loopvar3, %loopvar4, %loop2) => | |
| !c1_3 = I32 [1] | |
| !35 = ApplyBinaryPrimOp(%31, !c1_3) [Subtract] | |
| !36 = ApplyComparisonOp(%loopvar3, !35) [EQWithNA] | |
| !37 = ArrayRef(%30, %loopvar3) [-1] | |
| !38 = Cast(!37) [Int64] | |
| !39 = ApplyBinaryPrimOp(%loopvar4, !38) [Subtract] | |
| !40 = I64 [0] | |
| !41 = ApplyComparisonOp(!39, !40) [LTEQ] | |
| !42 = ApplySpecial(!36, !41) [-1, lor, | |
| (), Boolean] | |
| If !42 then { | |
| !c1_4 = I32 [1] | |
| !43 = ApplyBinaryPrimOp(%loopvar3, !c1_4) [Add] | |
| MakeTuple(!43, %loopvar4) [(0 1)] | |
| } else { | |
| !c1_5 = I32 [1] | |
| !44 = ApplyBinaryPrimOp(%loopvar3, !c1_5) [Add] | |
| !45 = ArrayRef(%30, %loopvar3) [-1] | |
| !46 = Cast(!45) [Int64] | |
| !47 = ApplyBinaryPrimOp(%loopvar4, !46) [Subtract] | |
| Recur(!44, !47) [Tuple[Int32,Int64]] | |
| } | |
| } | |
| } | |
| !s8 = ToStream(%3) [False] | |
| !49 = GetTupleElement(%48) [0] | |
| !s9 = StreamTake(!s8, !49) | |
| !c0_6 = I32 [0] | |
| !50 = GetTupleElement(%48) [0] | |
| !c1_6 = I32 [1] | |
| !s10 = StreamRange(!c0_6, !50, !c1_6) [-1, False] | |
| !s11 = StreamMap(!s10) { (%elt3) => | |
| !51 = GetTupleElement(%48) [0] | |
| !c1_7 = I32 [1] | |
| !52 = ApplyBinaryPrimOp(!51, !c1_7) [Subtract] | |
| !53 = ApplyComparisonOp(%elt3, !52) [EQWithNA] | |
| If !53 then { | |
| !54 = GetTupleElement(%48) [1] | |
| Cast(!54) [Int32] | |
| } else { | |
| ArrayRef(%30, %elt3) [-1] | |
| } | |
| } | |
| !s12 = StreamZip(!s9, !s11) [-1, AssumeSameLength] { | |
| (%elt4, %elt5) => | |
| MakeStruct(numberToTake: %elt5, old: %elt4) | |
| } | |
| ToArray(!s12) | |
| 2023-09-22 17:10:09.201 : INFO: Prune: MakeStruct: eliminating field '__iruid_3' | |
| 2023-09-22 17:10:09.332 : INFO: after optimize: relationalLowerer, initial IR: IR size 127: | |
| !c4 = I32 [4] | |
| !c0 = I32 [0] | |
| %22 = TailLoop(!c4, !c0) { | |
| (%loopvar, %loopvar2, %loop) => | |
| !1 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s = ToStream(!1) [False] | |
| !s2 = StreamTake(!s, %loopvar) | |
| !2 = Literal [Struct{}, <literal value>] | |
| !3 = Str ["iteration="] | |
| !4 = Apply(%loopvar2) [-1, str, (), String] | |
| !5 = Apply(!3, !4) [-1, concat, (), String] | |
| !6 = Str [",nParts="] | |
| !7 = Apply(!5, !6) [-1, concat, (), String] | |
| !8 = Apply(%loopvar) [-1, str, (), String] | |
| !9 = Apply(!7, !8) [-1, concat, (), String] | |
| %12 = CollectDistributedArray(!s2, !2, !9) { | |
| (%ctx, %g) => | |
| !10 = GetField(%ctx) [start] | |
| !11 = GetField(%ctx) [end] | |
| !c1 = I32 [1] | |
| !s3 = StreamRange(!10, !11, !c1) [-1, True] | |
| !s4 = StreamMap(!s3) { (%elt) => | |
| MakeStruct(idx: %elt) | |
| } | |
| !c11 = I32 [11] | |
| !s5 = StreamTake(!s4, !c11) | |
| StreamLen(!s5) | |
| } | |
| !s6 = ToStream(%12) [False] | |
| !c0_2 = I32 [0] | |
| !13 = StreamFold(!s6, !c0_2) { (%accum, %elt2) => | |
| ApplyBinaryPrimOp(%accum, %elt2) [Add] | |
| } | |
| !14 = Cast(!13) [Int64] | |
| !15 = I64 [11] | |
| !16 = ApplyComparisonOp(!14, !15) [GTEQ] | |
| !c2 = I32 [2] | |
| !17 = ArrayLen(%12) | |
| !18 = ApplyComparisonOp(!c2, !17) [LTEQ] | |
| !19 = ApplySpecial(!16, !18) [-1, lor, | |
| (), Boolean] | |
| If !19 then { %12 } else { | |
| !c4_2 = I32 [4] | |
| !20 = ApplyBinaryPrimOp(%loopvar, !c4_2) [Multiply] | |
| !c1_2 = I32 [1] | |
| !21 = ApplyBinaryPrimOp(%loopvar2, !c1_2) [Add] | |
| Recur(!20, !21) [Array[Int32]] | |
| } | |
| } | |
| %23 = ArrayLen(%22) | |
| !c0_3 = I32 [0] | |
| !24 = ApplyComparisonOp(%23, !c0_3) [EQWithNA] | |
| %39 = If !24 then { | |
| Literal [Tuple[Int32,Int64], <literal value>] | |
| } else { | |
| !c0_4 = I32 [0] | |
| !25 = I64 [11] | |
| TailLoop(!c0_4, !25) { | |
| (%loopvar3, %loopvar4, %loop2) => | |
| !c1_3 = I32 [1] | |
| !26 = ApplyBinaryPrimOp(%23, !c1_3) [Subtract] | |
| !27 = ApplyComparisonOp(%loopvar3, !26) [EQWithNA] | |
| !28 = ArrayRef(%22, %loopvar3) [-1] | |
| !29 = Cast(!28) [Int64] | |
| !30 = ApplyBinaryPrimOp(%loopvar4, !29) [Subtract] | |
| !31 = I64 [0] | |
| !32 = ApplyComparisonOp(!30, !31) [LTEQ] | |
| !33 = ApplySpecial(!27, !32) [-1, lor, | |
| (), Boolean] | |
| If !33 then { | |
| !c1_4 = I32 [1] | |
| !34 = ApplyBinaryPrimOp(%loopvar3, !c1_4) [Add] | |
| MakeTuple(!34, %loopvar4) [(0 1)] | |
| } else { | |
| !c1_5 = I32 [1] | |
| !35 = ApplyBinaryPrimOp(%loopvar3, !c1_5) [Add] | |
| !36 = ArrayRef(%22, %loopvar3) [-1] | |
| !37 = Cast(!36) [Int64] | |
| !38 = ApplyBinaryPrimOp(%loopvar4, !37) [Subtract] | |
| Recur(!35, !38) [Tuple[Int32,Int64]] | |
| } | |
| } | |
| } | |
| !40 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s7 = ToStream(!40) [False] | |
| !41 = GetTupleElement(%39) [0] | |
| !s8 = StreamTake(!s7, !41) | |
| !c0_5 = I32 [0] | |
| !42 = GetTupleElement(%39) [0] | |
| !c1_6 = I32 [1] | |
| !s9 = StreamRange(!c0_5, !42, !c1_6) [-1, False] | |
| !s10 = StreamMap(!s9) { (%elt3) => | |
| !43 = GetTupleElement(%39) [0] | |
| !c1_7 = I32 [1] | |
| !44 = ApplyBinaryPrimOp(!43, !c1_7) [Subtract] | |
| !45 = ApplyComparisonOp(%elt3, !44) [EQWithNA] | |
| If !45 then { | |
| !46 = GetTupleElement(%39) [1] | |
| Cast(!46) [Int32] | |
| } else { | |
| ArrayRef(%22, %elt3) [-1] | |
| } | |
| } | |
| !s11 = StreamZip(!s8, !s10) [-1, AssumeSameLength] { | |
| (%elt4, %elt5) => | |
| MakeStruct(numberToTake: %elt5, old: %elt4) | |
| } | |
| ToArray(!s11) | |
| 2023-09-22 17:10:09.364 : INFO: after LowerMatrixToTable: IR size 127: | |
| !c4 = I32 [4] | |
| !c0 = I32 [0] | |
| %22 = TailLoop(!c4, !c0) { | |
| (%loopvar, %loopvar2, %loop) => | |
| !1 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s = ToStream(!1) [False] | |
| !s2 = StreamTake(!s, %loopvar) | |
| !2 = Literal [Struct{}, <literal value>] | |
| !3 = Str ["iteration="] | |
| !4 = Apply(%loopvar2) [-1, str, (), String] | |
| !5 = Apply(!3, !4) [-1, concat, (), String] | |
| !6 = Str [",nParts="] | |
| !7 = Apply(!5, !6) [-1, concat, (), String] | |
| !8 = Apply(%loopvar) [-1, str, (), String] | |
| !9 = Apply(!7, !8) [-1, concat, (), String] | |
| %12 = CollectDistributedArray(!s2, !2, !9) { | |
| (%ctx, %g) => | |
| !10 = GetField(%ctx) [start] | |
| !11 = GetField(%ctx) [end] | |
| !c1 = I32 [1] | |
| !s3 = StreamRange(!10, !11, !c1) [-1, True] | |
| !s4 = StreamMap(!s3) { (%elt) => | |
| MakeStruct(idx: %elt) | |
| } | |
| !c11 = I32 [11] | |
| !s5 = StreamTake(!s4, !c11) | |
| StreamLen(!s5) | |
| } | |
| !s6 = ToStream(%12) [False] | |
| !c0_2 = I32 [0] | |
| !13 = StreamFold(!s6, !c0_2) { (%accum, %elt2) => | |
| ApplyBinaryPrimOp(%accum, %elt2) [Add] | |
| } | |
| !14 = Cast(!13) [Int64] | |
| !15 = I64 [11] | |
| !16 = ApplyComparisonOp(!14, !15) [GTEQ] | |
| !c2 = I32 [2] | |
| !17 = ArrayLen(%12) | |
| !18 = ApplyComparisonOp(!c2, !17) [LTEQ] | |
| !19 = ApplySpecial(!16, !18) [-1, lor, | |
| (), Boolean] | |
| If !19 then { %12 } else { | |
| !c4_2 = I32 [4] | |
| !20 = ApplyBinaryPrimOp(%loopvar, !c4_2) [Multiply] | |
| !c1_2 = I32 [1] | |
| !21 = ApplyBinaryPrimOp(%loopvar2, !c1_2) [Add] | |
| Recur(!20, !21) [Array[Int32]] | |
| } | |
| } | |
| %23 = ArrayLen(%22) | |
| !c0_3 = I32 [0] | |
| !24 = ApplyComparisonOp(%23, !c0_3) [EQWithNA] | |
| %39 = If !24 then { | |
| Literal [Tuple[Int32,Int64], <literal value>] | |
| } else { | |
| !c0_4 = I32 [0] | |
| !25 = I64 [11] | |
| TailLoop(!c0_4, !25) { | |
| (%loopvar3, %loopvar4, %loop2) => | |
| !c1_3 = I32 [1] | |
| !26 = ApplyBinaryPrimOp(%23, !c1_3) [Subtract] | |
| !27 = ApplyComparisonOp(%loopvar3, !26) [EQWithNA] | |
| !28 = ArrayRef(%22, %loopvar3) [-1] | |
| !29 = Cast(!28) [Int64] | |
| !30 = ApplyBinaryPrimOp(%loopvar4, !29) [Subtract] | |
| !31 = I64 [0] | |
| !32 = ApplyComparisonOp(!30, !31) [LTEQ] | |
| !33 = ApplySpecial(!27, !32) [-1, lor, | |
| (), Boolean] | |
| If !33 then { | |
| !c1_4 = I32 [1] | |
| !34 = ApplyBinaryPrimOp(%loopvar3, !c1_4) [Add] | |
| MakeTuple(!34, %loopvar4) [(0 1)] | |
| } else { | |
| !c1_5 = I32 [1] | |
| !35 = ApplyBinaryPrimOp(%loopvar3, !c1_5) [Add] | |
| !36 = ArrayRef(%22, %loopvar3) [-1] | |
| !37 = Cast(!36) [Int64] | |
| !38 = ApplyBinaryPrimOp(%loopvar4, !37) [Subtract] | |
| Recur(!35, !38) [Tuple[Int32,Int64]] | |
| } | |
| } | |
| } | |
| !40 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s7 = ToStream(!40) [False] | |
| !41 = GetTupleElement(%39) [0] | |
| !s8 = StreamTake(!s7, !41) | |
| !c0_5 = I32 [0] | |
| !42 = GetTupleElement(%39) [0] | |
| !c1_6 = I32 [1] | |
| !s9 = StreamRange(!c0_5, !42, !c1_6) [-1, False] | |
| !s10 = StreamMap(!s9) { (%elt3) => | |
| !43 = GetTupleElement(%39) [0] | |
| !c1_7 = I32 [1] | |
| !44 = ApplyBinaryPrimOp(!43, !c1_7) [Subtract] | |
| !45 = ApplyComparisonOp(%elt3, !44) [EQWithNA] | |
| If !45 then { | |
| !46 = GetTupleElement(%39) [1] | |
| Cast(!46) [Int32] | |
| } else { | |
| ArrayRef(%22, %elt3) [-1] | |
| } | |
| } | |
| !s11 = StreamZip(!s8, !s10) [-1, AssumeSameLength] { | |
| (%elt4, %elt5) => | |
| MakeStruct(numberToTake: %elt5, old: %elt4) | |
| } | |
| ToArray(!s11) | |
| 2023-09-22 17:10:09.492 : INFO: after optimize: relationalLowerer, after LowerMatrixToTable: IR size 127: | |
| !c4 = I32 [4] | |
| !c0 = I32 [0] | |
| %22 = TailLoop(!c4, !c0) { | |
| (%loopvar, %loopvar2, %loop) => | |
| !1 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s = ToStream(!1) [False] | |
| !s2 = StreamTake(!s, %loopvar) | |
| !2 = Literal [Struct{}, <literal value>] | |
| !3 = Str ["iteration="] | |
| !4 = Apply(%loopvar2) [-1, str, (), String] | |
| !5 = Apply(!3, !4) [-1, concat, (), String] | |
| !6 = Str [",nParts="] | |
| !7 = Apply(!5, !6) [-1, concat, (), String] | |
| !8 = Apply(%loopvar) [-1, str, (), String] | |
| !9 = Apply(!7, !8) [-1, concat, (), String] | |
| %12 = CollectDistributedArray(!s2, !2, !9) { | |
| (%ctx, %g) => | |
| !10 = GetField(%ctx) [start] | |
| !11 = GetField(%ctx) [end] | |
| !c1 = I32 [1] | |
| !s3 = StreamRange(!10, !11, !c1) [-1, True] | |
| !s4 = StreamMap(!s3) { (%elt) => | |
| MakeStruct(idx: %elt) | |
| } | |
| !c11 = I32 [11] | |
| !s5 = StreamTake(!s4, !c11) | |
| StreamLen(!s5) | |
| } | |
| !s6 = ToStream(%12) [False] | |
| !c0_2 = I32 [0] | |
| !13 = StreamFold(!s6, !c0_2) { (%accum, %elt2) => | |
| ApplyBinaryPrimOp(%accum, %elt2) [Add] | |
| } | |
| !14 = Cast(!13) [Int64] | |
| !15 = I64 [11] | |
| !16 = ApplyComparisonOp(!14, !15) [GTEQ] | |
| !c2 = I32 [2] | |
| !17 = ArrayLen(%12) | |
| !18 = ApplyComparisonOp(!c2, !17) [LTEQ] | |
| !19 = ApplySpecial(!16, !18) [-1, lor, | |
| (), Boolean] | |
| If !19 then { %12 } else { | |
| !c4_2 = I32 [4] | |
| !20 = ApplyBinaryPrimOp(%loopvar, !c4_2) [Multiply] | |
| !c1_2 = I32 [1] | |
| !21 = ApplyBinaryPrimOp(%loopvar2, !c1_2) [Add] | |
| Recur(!20, !21) [Array[Int32]] | |
| } | |
| } | |
| %23 = ArrayLen(%22) | |
| !c0_3 = I32 [0] | |
| !24 = ApplyComparisonOp(%23, !c0_3) [EQWithNA] | |
| %39 = If !24 then { | |
| Literal [Tuple[Int32,Int64], <literal value>] | |
| } else { | |
| !c0_4 = I32 [0] | |
| !25 = I64 [11] | |
| TailLoop(!c0_4, !25) { | |
| (%loopvar3, %loopvar4, %loop2) => | |
| !c1_3 = I32 [1] | |
| !26 = ApplyBinaryPrimOp(%23, !c1_3) [Subtract] | |
| !27 = ApplyComparisonOp(%loopvar3, !26) [EQWithNA] | |
| !28 = ArrayRef(%22, %loopvar3) [-1] | |
| !29 = Cast(!28) [Int64] | |
| !30 = ApplyBinaryPrimOp(%loopvar4, !29) [Subtract] | |
| !31 = I64 [0] | |
| !32 = ApplyComparisonOp(!30, !31) [LTEQ] | |
| !33 = ApplySpecial(!27, !32) [-1, lor, | |
| (), Boolean] | |
| If !33 then { | |
| !c1_4 = I32 [1] | |
| !34 = ApplyBinaryPrimOp(%loopvar3, !c1_4) [Add] | |
| MakeTuple(!34, %loopvar4) [(0 1)] | |
| } else { | |
| !c1_5 = I32 [1] | |
| !35 = ApplyBinaryPrimOp(%loopvar3, !c1_5) [Add] | |
| !36 = ArrayRef(%22, %loopvar3) [-1] | |
| !37 = Cast(!36) [Int64] | |
| !38 = ApplyBinaryPrimOp(%loopvar4, !37) [Subtract] | |
| Recur(!35, !38) [Tuple[Int32,Int64]] | |
| } | |
| } | |
| } | |
| !40 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s7 = ToStream(!40) [False] | |
| !41 = GetTupleElement(%39) [0] | |
| !s8 = StreamTake(!s7, !41) | |
| !c0_5 = I32 [0] | |
| !42 = GetTupleElement(%39) [0] | |
| !c1_6 = I32 [1] | |
| !s9 = StreamRange(!c0_5, !42, !c1_6) [-1, False] | |
| !s10 = StreamMap(!s9) { (%elt3) => | |
| !43 = GetTupleElement(%39) [0] | |
| !c1_7 = I32 [1] | |
| !44 = ApplyBinaryPrimOp(!43, !c1_7) [Subtract] | |
| !45 = ApplyComparisonOp(%elt3, !44) [EQWithNA] | |
| If !45 then { | |
| !46 = GetTupleElement(%39) [1] | |
| Cast(!46) [Int32] | |
| } else { | |
| ArrayRef(%22, %elt3) [-1] | |
| } | |
| } | |
| !s11 = StreamZip(!s8, !s10) [-1, AssumeSameLength] { | |
| (%elt4, %elt5) => | |
| MakeStruct(numberToTake: %elt5, old: %elt4) | |
| } | |
| ToArray(!s11) | |
| 2023-09-22 17:10:09.506 : INFO: after LiftRelationalValuesToRelationalLets: IR size 127: | |
| !c4 = I32 [4] | |
| !c0 = I32 [0] | |
| %22 = TailLoop(!c4, !c0) { | |
| (%loopvar, %loopvar2, %loop) => | |
| !1 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s = ToStream(!1) [False] | |
| !s2 = StreamTake(!s, %loopvar) | |
| !2 = Literal [Struct{}, <literal value>] | |
| !3 = Str ["iteration="] | |
| !4 = Apply(%loopvar2) [-1, str, (), String] | |
| !5 = Apply(!3, !4) [-1, concat, (), String] | |
| !6 = Str [",nParts="] | |
| !7 = Apply(!5, !6) [-1, concat, (), String] | |
| !8 = Apply(%loopvar) [-1, str, (), String] | |
| !9 = Apply(!7, !8) [-1, concat, (), String] | |
| %12 = CollectDistributedArray(!s2, !2, !9) { | |
| (%ctx, %g) => | |
| !10 = GetField(%ctx) [start] | |
| !11 = GetField(%ctx) [end] | |
| !c1 = I32 [1] | |
| !s3 = StreamRange(!10, !11, !c1) [-1, True] | |
| !s4 = StreamMap(!s3) { (%elt) => | |
| MakeStruct(idx: %elt) | |
| } | |
| !c11 = I32 [11] | |
| !s5 = StreamTake(!s4, !c11) | |
| StreamLen(!s5) | |
| } | |
| !s6 = ToStream(%12) [False] | |
| !c0_2 = I32 [0] | |
| !13 = StreamFold(!s6, !c0_2) { (%accum, %elt2) => | |
| ApplyBinaryPrimOp(%accum, %elt2) [Add] | |
| } | |
| !14 = Cast(!13) [Int64] | |
| !15 = I64 [11] | |
| !16 = ApplyComparisonOp(!14, !15) [GTEQ] | |
| !c2 = I32 [2] | |
| !17 = ArrayLen(%12) | |
| !18 = ApplyComparisonOp(!c2, !17) [LTEQ] | |
| !19 = ApplySpecial(!16, !18) [-1, lor, | |
| (), Boolean] | |
| If !19 then { %12 } else { | |
| !c4_2 = I32 [4] | |
| !20 = ApplyBinaryPrimOp(%loopvar, !c4_2) [Multiply] | |
| !c1_2 = I32 [1] | |
| !21 = ApplyBinaryPrimOp(%loopvar2, !c1_2) [Add] | |
| Recur(!20, !21) [Array[Int32]] | |
| } | |
| } | |
| %23 = ArrayLen(%22) | |
| !c0_3 = I32 [0] | |
| !24 = ApplyComparisonOp(%23, !c0_3) [EQWithNA] | |
| %39 = If !24 then { | |
| Literal [Tuple[Int32,Int64], <literal value>] | |
| } else { | |
| !c0_4 = I32 [0] | |
| !25 = I64 [11] | |
| TailLoop(!c0_4, !25) { | |
| (%loopvar3, %loopvar4, %loop2) => | |
| !c1_3 = I32 [1] | |
| !26 = ApplyBinaryPrimOp(%23, !c1_3) [Subtract] | |
| !27 = ApplyComparisonOp(%loopvar3, !26) [EQWithNA] | |
| !28 = ArrayRef(%22, %loopvar3) [-1] | |
| !29 = Cast(!28) [Int64] | |
| !30 = ApplyBinaryPrimOp(%loopvar4, !29) [Subtract] | |
| !31 = I64 [0] | |
| !32 = ApplyComparisonOp(!30, !31) [LTEQ] | |
| !33 = ApplySpecial(!27, !32) [-1, lor, | |
| (), Boolean] | |
| If !33 then { | |
| !c1_4 = I32 [1] | |
| !34 = ApplyBinaryPrimOp(%loopvar3, !c1_4) [Add] | |
| MakeTuple(!34, %loopvar4) [(0 1)] | |
| } else { | |
| !c1_5 = I32 [1] | |
| !35 = ApplyBinaryPrimOp(%loopvar3, !c1_5) [Add] | |
| !36 = ArrayRef(%22, %loopvar3) [-1] | |
| !37 = Cast(!36) [Int64] | |
| !38 = ApplyBinaryPrimOp(%loopvar4, !37) [Subtract] | |
| Recur(!35, !38) [Tuple[Int32,Int64]] | |
| } | |
| } | |
| } | |
| !40 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s7 = ToStream(!40) [False] | |
| !41 = GetTupleElement(%39) [0] | |
| !s8 = StreamTake(!s7, !41) | |
| !c0_5 = I32 [0] | |
| !42 = GetTupleElement(%39) [0] | |
| !c1_6 = I32 [1] | |
| !s9 = StreamRange(!c0_5, !42, !c1_6) [-1, False] | |
| !s10 = StreamMap(!s9) { (%elt3) => | |
| !43 = GetTupleElement(%39) [0] | |
| !c1_7 = I32 [1] | |
| !44 = ApplyBinaryPrimOp(!43, !c1_7) [Subtract] | |
| !45 = ApplyComparisonOp(%elt3, !44) [EQWithNA] | |
| If !45 then { | |
| !46 = GetTupleElement(%39) [1] | |
| Cast(!46) [Int32] | |
| } else { | |
| ArrayRef(%22, %elt3) [-1] | |
| } | |
| } | |
| !s11 = StreamZip(!s8, !s10) [-1, AssumeSameLength] { | |
| (%elt4, %elt5) => | |
| MakeStruct(numberToTake: %elt5, old: %elt4) | |
| } | |
| ToArray(!s11) | |
| 2023-09-22 17:10:09.522 : INFO: after EvalRelationalLets: IR size 127: | |
| !c4 = I32 [4] | |
| !c0 = I32 [0] | |
| %22 = TailLoop(!c4, !c0) { | |
| (%loopvar, %loopvar2, %loop) => | |
| !1 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s = ToStream(!1) [False] | |
| !s2 = StreamTake(!s, %loopvar) | |
| !2 = Literal [Struct{}, <literal value>] | |
| !3 = Str ["iteration="] | |
| !4 = Apply(%loopvar2) [-1, str, (), String] | |
| !5 = Apply(!3, !4) [-1, concat, (), String] | |
| !6 = Str [",nParts="] | |
| !7 = Apply(!5, !6) [-1, concat, (), String] | |
| !8 = Apply(%loopvar) [-1, str, (), String] | |
| !9 = Apply(!7, !8) [-1, concat, (), String] | |
| %12 = CollectDistributedArray(!s2, !2, !9) { | |
| (%ctx, %g) => | |
| !10 = GetField(%ctx) [start] | |
| !11 = GetField(%ctx) [end] | |
| !c1 = I32 [1] | |
| !s3 = StreamRange(!10, !11, !c1) [-1, True] | |
| !s4 = StreamMap(!s3) { (%elt) => | |
| MakeStruct(idx: %elt) | |
| } | |
| !c11 = I32 [11] | |
| !s5 = StreamTake(!s4, !c11) | |
| StreamLen(!s5) | |
| } | |
| !s6 = ToStream(%12) [False] | |
| !c0_2 = I32 [0] | |
| !13 = StreamFold(!s6, !c0_2) { (%accum, %elt2) => | |
| ApplyBinaryPrimOp(%accum, %elt2) [Add] | |
| } | |
| !14 = Cast(!13) [Int64] | |
| !15 = I64 [11] | |
| !16 = ApplyComparisonOp(!14, !15) [GTEQ] | |
| !c2 = I32 [2] | |
| !17 = ArrayLen(%12) | |
| !18 = ApplyComparisonOp(!c2, !17) [LTEQ] | |
| !19 = ApplySpecial(!16, !18) [-1, lor, | |
| (), Boolean] | |
| If !19 then { %12 } else { | |
| !c4_2 = I32 [4] | |
| !20 = ApplyBinaryPrimOp(%loopvar, !c4_2) [Multiply] | |
| !c1_2 = I32 [1] | |
| !21 = ApplyBinaryPrimOp(%loopvar2, !c1_2) [Add] | |
| Recur(!20, !21) [Array[Int32]] | |
| } | |
| } | |
| %23 = ArrayLen(%22) | |
| !c0_3 = I32 [0] | |
| !24 = ApplyComparisonOp(%23, !c0_3) [EQWithNA] | |
| %39 = If !24 then { | |
| Literal [Tuple[Int32,Int64], <literal value>] | |
| } else { | |
| !c0_4 = I32 [0] | |
| !25 = I64 [11] | |
| TailLoop(!c0_4, !25) { | |
| (%loopvar3, %loopvar4, %loop2) => | |
| !c1_3 = I32 [1] | |
| !26 = ApplyBinaryPrimOp(%23, !c1_3) [Subtract] | |
| !27 = ApplyComparisonOp(%loopvar3, !26) [EQWithNA] | |
| !28 = ArrayRef(%22, %loopvar3) [-1] | |
| !29 = Cast(!28) [Int64] | |
| !30 = ApplyBinaryPrimOp(%loopvar4, !29) [Subtract] | |
| !31 = I64 [0] | |
| !32 = ApplyComparisonOp(!30, !31) [LTEQ] | |
| !33 = ApplySpecial(!27, !32) [-1, lor, | |
| (), Boolean] | |
| If !33 then { | |
| !c1_4 = I32 [1] | |
| !34 = ApplyBinaryPrimOp(%loopvar3, !c1_4) [Add] | |
| MakeTuple(!34, %loopvar4) [(0 1)] | |
| } else { | |
| !c1_5 = I32 [1] | |
| !35 = ApplyBinaryPrimOp(%loopvar3, !c1_5) [Add] | |
| !36 = ArrayRef(%22, %loopvar3) [-1] | |
| !37 = Cast(!36) [Int64] | |
| !38 = ApplyBinaryPrimOp(%loopvar4, !37) [Subtract] | |
| Recur(!35, !38) [Tuple[Int32,Int64]] | |
| } | |
| } | |
| } | |
| !40 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s7 = ToStream(!40) [False] | |
| !41 = GetTupleElement(%39) [0] | |
| !s8 = StreamTake(!s7, !41) | |
| !c0_5 = I32 [0] | |
| !42 = GetTupleElement(%39) [0] | |
| !c1_6 = I32 [1] | |
| !s9 = StreamRange(!c0_5, !42, !c1_6) [-1, False] | |
| !s10 = StreamMap(!s9) { (%elt3) => | |
| !43 = GetTupleElement(%39) [0] | |
| !c1_7 = I32 [1] | |
| !44 = ApplyBinaryPrimOp(!43, !c1_7) [Subtract] | |
| !45 = ApplyComparisonOp(%elt3, !44) [EQWithNA] | |
| If !45 then { | |
| !46 = GetTupleElement(%39) [1] | |
| Cast(!46) [Int32] | |
| } else { | |
| ArrayRef(%22, %elt3) [-1] | |
| } | |
| } | |
| !s11 = StreamZip(!s8, !s10) [-1, AssumeSameLength] { | |
| (%elt4, %elt5) => | |
| MakeStruct(numberToTake: %elt5, old: %elt4) | |
| } | |
| ToArray(!s11) | |
| 2023-09-22 17:10:09.532 : INFO: after LowerAndExecuteShuffles: IR size 127: | |
| !c4 = I32 [4] | |
| !c0 = I32 [0] | |
| %22 = TailLoop(!c4, !c0) { | |
| (%loopvar, %loopvar2, %loop) => | |
| !1 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s = ToStream(!1) [False] | |
| !s2 = StreamTake(!s, %loopvar) | |
| !2 = Literal [Struct{}, <literal value>] | |
| !3 = Str ["iteration="] | |
| !4 = Apply(%loopvar2) [-1, str, (), String] | |
| !5 = Apply(!3, !4) [-1, concat, (), String] | |
| !6 = Str [",nParts="] | |
| !7 = Apply(!5, !6) [-1, concat, (), String] | |
| !8 = Apply(%loopvar) [-1, str, (), String] | |
| !9 = Apply(!7, !8) [-1, concat, (), String] | |
| %12 = CollectDistributedArray(!s2, !2, !9) { | |
| (%ctx, %g) => | |
| !10 = GetField(%ctx) [start] | |
| !11 = GetField(%ctx) [end] | |
| !c1 = I32 [1] | |
| !s3 = StreamRange(!10, !11, !c1) [-1, True] | |
| !s4 = StreamMap(!s3) { (%elt) => | |
| MakeStruct(idx: %elt) | |
| } | |
| !c11 = I32 [11] | |
| !s5 = StreamTake(!s4, !c11) | |
| StreamLen(!s5) | |
| } | |
| !s6 = ToStream(%12) [False] | |
| !c0_2 = I32 [0] | |
| !13 = StreamFold(!s6, !c0_2) { (%accum, %elt2) => | |
| ApplyBinaryPrimOp(%accum, %elt2) [Add] | |
| } | |
| !14 = Cast(!13) [Int64] | |
| !15 = I64 [11] | |
| !16 = ApplyComparisonOp(!14, !15) [GTEQ] | |
| !c2 = I32 [2] | |
| !17 = ArrayLen(%12) | |
| !18 = ApplyComparisonOp(!c2, !17) [LTEQ] | |
| !19 = ApplySpecial(!16, !18) [-1, lor, | |
| (), Boolean] | |
| If !19 then { %12 } else { | |
| !c4_2 = I32 [4] | |
| !20 = ApplyBinaryPrimOp(%loopvar, !c4_2) [Multiply] | |
| !c1_2 = I32 [1] | |
| !21 = ApplyBinaryPrimOp(%loopvar2, !c1_2) [Add] | |
| Recur(!20, !21) [Array[Int32]] | |
| } | |
| } | |
| %23 = ArrayLen(%22) | |
| !c0_3 = I32 [0] | |
| !24 = ApplyComparisonOp(%23, !c0_3) [EQWithNA] | |
| %39 = If !24 then { | |
| Literal [Tuple[Int32,Int64], <literal value>] | |
| } else { | |
| !c0_4 = I32 [0] | |
| !25 = I64 [11] | |
| TailLoop(!c0_4, !25) { | |
| (%loopvar3, %loopvar4, %loop2) => | |
| !c1_3 = I32 [1] | |
| !26 = ApplyBinaryPrimOp(%23, !c1_3) [Subtract] | |
| !27 = ApplyComparisonOp(%loopvar3, !26) [EQWithNA] | |
| !28 = ArrayRef(%22, %loopvar3) [-1] | |
| !29 = Cast(!28) [Int64] | |
| !30 = ApplyBinaryPrimOp(%loopvar4, !29) [Subtract] | |
| !31 = I64 [0] | |
| !32 = ApplyComparisonOp(!30, !31) [LTEQ] | |
| !33 = ApplySpecial(!27, !32) [-1, lor, | |
| (), Boolean] | |
| If !33 then { | |
| !c1_4 = I32 [1] | |
| !34 = ApplyBinaryPrimOp(%loopvar3, !c1_4) [Add] | |
| MakeTuple(!34, %loopvar4) [(0 1)] | |
| } else { | |
| !c1_5 = I32 [1] | |
| !35 = ApplyBinaryPrimOp(%loopvar3, !c1_5) [Add] | |
| !36 = ArrayRef(%22, %loopvar3) [-1] | |
| !37 = Cast(!36) [Int64] | |
| !38 = ApplyBinaryPrimOp(%loopvar4, !37) [Subtract] | |
| Recur(!35, !38) [Tuple[Int32,Int64]] | |
| } | |
| } | |
| } | |
| !40 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s7 = ToStream(!40) [False] | |
| !41 = GetTupleElement(%39) [0] | |
| !s8 = StreamTake(!s7, !41) | |
| !c0_5 = I32 [0] | |
| !42 = GetTupleElement(%39) [0] | |
| !c1_6 = I32 [1] | |
| !s9 = StreamRange(!c0_5, !42, !c1_6) [-1, False] | |
| !s10 = StreamMap(!s9) { (%elt3) => | |
| !43 = GetTupleElement(%39) [0] | |
| !c1_7 = I32 [1] | |
| !44 = ApplyBinaryPrimOp(!43, !c1_7) [Subtract] | |
| !45 = ApplyComparisonOp(%elt3, !44) [EQWithNA] | |
| If !45 then { | |
| !46 = GetTupleElement(%39) [1] | |
| Cast(!46) [Int32] | |
| } else { | |
| ArrayRef(%22, %elt3) [-1] | |
| } | |
| } | |
| !s11 = StreamZip(!s8, !s10) [-1, AssumeSameLength] { | |
| (%elt4, %elt5) => | |
| MakeStruct(numberToTake: %elt5, old: %elt4) | |
| } | |
| ToArray(!s11) | |
| 2023-09-22 17:10:09.625 : INFO: after optimize: relationalLowerer, after LowerAndExecuteShuffles: IR size 127: | |
| !c4 = I32 [4] | |
| !c0 = I32 [0] | |
| %22 = TailLoop(!c4, !c0) { | |
| (%loopvar, %loopvar2, %loop) => | |
| !1 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s = ToStream(!1) [False] | |
| !s2 = StreamTake(!s, %loopvar) | |
| !2 = Literal [Struct{}, <literal value>] | |
| !3 = Str ["iteration="] | |
| !4 = Apply(%loopvar2) [-1, str, (), String] | |
| !5 = Apply(!3, !4) [-1, concat, (), String] | |
| !6 = Str [",nParts="] | |
| !7 = Apply(!5, !6) [-1, concat, (), String] | |
| !8 = Apply(%loopvar) [-1, str, (), String] | |
| !9 = Apply(!7, !8) [-1, concat, (), String] | |
| %12 = CollectDistributedArray(!s2, !2, !9) { | |
| (%ctx, %g) => | |
| !10 = GetField(%ctx) [start] | |
| !11 = GetField(%ctx) [end] | |
| !c1 = I32 [1] | |
| !s3 = StreamRange(!10, !11, !c1) [-1, True] | |
| !s4 = StreamMap(!s3) { (%elt) => | |
| MakeStruct(idx: %elt) | |
| } | |
| !c11 = I32 [11] | |
| !s5 = StreamTake(!s4, !c11) | |
| StreamLen(!s5) | |
| } | |
| !s6 = ToStream(%12) [False] | |
| !c0_2 = I32 [0] | |
| !13 = StreamFold(!s6, !c0_2) { (%accum, %elt2) => | |
| ApplyBinaryPrimOp(%accum, %elt2) [Add] | |
| } | |
| !14 = Cast(!13) [Int64] | |
| !15 = I64 [11] | |
| !16 = ApplyComparisonOp(!14, !15) [GTEQ] | |
| !c2 = I32 [2] | |
| !17 = ArrayLen(%12) | |
| !18 = ApplyComparisonOp(!c2, !17) [LTEQ] | |
| !19 = ApplySpecial(!16, !18) [-1, lor, | |
| (), Boolean] | |
| If !19 then { %12 } else { | |
| !c4_2 = I32 [4] | |
| !20 = ApplyBinaryPrimOp(%loopvar, !c4_2) [Multiply] | |
| !c1_2 = I32 [1] | |
| !21 = ApplyBinaryPrimOp(%loopvar2, !c1_2) [Add] | |
| Recur(!20, !21) [Array[Int32]] | |
| } | |
| } | |
| %23 = ArrayLen(%22) | |
| !c0_3 = I32 [0] | |
| !24 = ApplyComparisonOp(%23, !c0_3) [EQWithNA] | |
| %39 = If !24 then { | |
| Literal [Tuple[Int32,Int64], <literal value>] | |
| } else { | |
| !c0_4 = I32 [0] | |
| !25 = I64 [11] | |
| TailLoop(!c0_4, !25) { | |
| (%loopvar3, %loopvar4, %loop2) => | |
| !c1_3 = I32 [1] | |
| !26 = ApplyBinaryPrimOp(%23, !c1_3) [Subtract] | |
| !27 = ApplyComparisonOp(%loopvar3, !26) [EQWithNA] | |
| !28 = ArrayRef(%22, %loopvar3) [-1] | |
| !29 = Cast(!28) [Int64] | |
| !30 = ApplyBinaryPrimOp(%loopvar4, !29) [Subtract] | |
| !31 = I64 [0] | |
| !32 = ApplyComparisonOp(!30, !31) [LTEQ] | |
| !33 = ApplySpecial(!27, !32) [-1, lor, | |
| (), Boolean] | |
| If !33 then { | |
| !c1_4 = I32 [1] | |
| !34 = ApplyBinaryPrimOp(%loopvar3, !c1_4) [Add] | |
| MakeTuple(!34, %loopvar4) [(0 1)] | |
| } else { | |
| !c1_5 = I32 [1] | |
| !35 = ApplyBinaryPrimOp(%loopvar3, !c1_5) [Add] | |
| !36 = ArrayRef(%22, %loopvar3) [-1] | |
| !37 = Cast(!36) [Int64] | |
| !38 = ApplyBinaryPrimOp(%loopvar4, !37) [Subtract] | |
| Recur(!35, !38) [Tuple[Int32,Int64]] | |
| } | |
| } | |
| } | |
| !40 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s7 = ToStream(!40) [False] | |
| !41 = GetTupleElement(%39) [0] | |
| !s8 = StreamTake(!s7, !41) | |
| !c0_5 = I32 [0] | |
| !42 = GetTupleElement(%39) [0] | |
| !c1_6 = I32 [1] | |
| !s9 = StreamRange(!c0_5, !42, !c1_6) [-1, False] | |
| !s10 = StreamMap(!s9) { (%elt3) => | |
| !43 = GetTupleElement(%39) [0] | |
| !c1_7 = I32 [1] | |
| !44 = ApplyBinaryPrimOp(!43, !c1_7) [Subtract] | |
| !45 = ApplyComparisonOp(%elt3, !44) [EQWithNA] | |
| If !45 then { | |
| !46 = GetTupleElement(%39) [1] | |
| Cast(!46) [Int32] | |
| } else { | |
| ArrayRef(%22, %elt3) [-1] | |
| } | |
| } | |
| !s11 = StreamZip(!s8, !s10) [-1, AssumeSameLength] { | |
| (%elt4, %elt5) => | |
| MakeStruct(numberToTake: %elt5, old: %elt4) | |
| } | |
| ToArray(!s11) | |
| 2023-09-22 17:10:09.637 : INFO: after LowerOrInterpretNonCompilable: IR size 127: | |
| !c4 = I32 [4] | |
| !c0 = I32 [0] | |
| %22 = TailLoop(!c4, !c0) { | |
| (%loopvar, %loopvar2, %loop) => | |
| !1 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s = ToStream(!1) [False] | |
| !s2 = StreamTake(!s, %loopvar) | |
| !2 = Literal [Struct{}, <literal value>] | |
| !3 = Str ["iteration="] | |
| !4 = Apply(%loopvar2) [-1, str, (), String] | |
| !5 = Apply(!3, !4) [-1, concat, (), String] | |
| !6 = Str [",nParts="] | |
| !7 = Apply(!5, !6) [-1, concat, (), String] | |
| !8 = Apply(%loopvar) [-1, str, (), String] | |
| !9 = Apply(!7, !8) [-1, concat, (), String] | |
| %12 = CollectDistributedArray(!s2, !2, !9) { | |
| (%ctx, %g) => | |
| !10 = GetField(%ctx) [start] | |
| !11 = GetField(%ctx) [end] | |
| !c1 = I32 [1] | |
| !s3 = StreamRange(!10, !11, !c1) [-1, True] | |
| !s4 = StreamMap(!s3) { (%elt) => | |
| MakeStruct(idx: %elt) | |
| } | |
| !c11 = I32 [11] | |
| !s5 = StreamTake(!s4, !c11) | |
| StreamLen(!s5) | |
| } | |
| !s6 = ToStream(%12) [False] | |
| !c0_2 = I32 [0] | |
| !13 = StreamFold(!s6, !c0_2) { (%accum, %elt2) => | |
| ApplyBinaryPrimOp(%accum, %elt2) [Add] | |
| } | |
| !14 = Cast(!13) [Int64] | |
| !15 = I64 [11] | |
| !16 = ApplyComparisonOp(!14, !15) [GTEQ] | |
| !c2 = I32 [2] | |
| !17 = ArrayLen(%12) | |
| !18 = ApplyComparisonOp(!c2, !17) [LTEQ] | |
| !19 = ApplySpecial(!16, !18) [-1, lor, | |
| (), Boolean] | |
| If !19 then { %12 } else { | |
| !c4_2 = I32 [4] | |
| !20 = ApplyBinaryPrimOp(%loopvar, !c4_2) [Multiply] | |
| !c1_2 = I32 [1] | |
| !21 = ApplyBinaryPrimOp(%loopvar2, !c1_2) [Add] | |
| Recur(!20, !21) [Array[Int32]] | |
| } | |
| } | |
| %23 = ArrayLen(%22) | |
| !c0_3 = I32 [0] | |
| !24 = ApplyComparisonOp(%23, !c0_3) [EQWithNA] | |
| %39 = If !24 then { | |
| Literal [Tuple[Int32,Int64], <literal value>] | |
| } else { | |
| !c0_4 = I32 [0] | |
| !25 = I64 [11] | |
| TailLoop(!c0_4, !25) { | |
| (%loopvar3, %loopvar4, %loop2) => | |
| !c1_3 = I32 [1] | |
| !26 = ApplyBinaryPrimOp(%23, !c1_3) [Subtract] | |
| !27 = ApplyComparisonOp(%loopvar3, !26) [EQWithNA] | |
| !28 = ArrayRef(%22, %loopvar3) [-1] | |
| !29 = Cast(!28) [Int64] | |
| !30 = ApplyBinaryPrimOp(%loopvar4, !29) [Subtract] | |
| !31 = I64 [0] | |
| !32 = ApplyComparisonOp(!30, !31) [LTEQ] | |
| !33 = ApplySpecial(!27, !32) [-1, lor, | |
| (), Boolean] | |
| If !33 then { | |
| !c1_4 = I32 [1] | |
| !34 = ApplyBinaryPrimOp(%loopvar3, !c1_4) [Add] | |
| MakeTuple(!34, %loopvar4) [(0 1)] | |
| } else { | |
| !c1_5 = I32 [1] | |
| !35 = ApplyBinaryPrimOp(%loopvar3, !c1_5) [Add] | |
| !36 = ArrayRef(%22, %loopvar3) [-1] | |
| !37 = Cast(!36) [Int64] | |
| !38 = ApplyBinaryPrimOp(%loopvar4, !37) [Subtract] | |
| Recur(!35, !38) [Tuple[Int32,Int64]] | |
| } | |
| } | |
| } | |
| !40 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s7 = ToStream(!40) [False] | |
| !41 = GetTupleElement(%39) [0] | |
| !s8 = StreamTake(!s7, !41) | |
| !c0_5 = I32 [0] | |
| !42 = GetTupleElement(%39) [0] | |
| !c1_6 = I32 [1] | |
| !s9 = StreamRange(!c0_5, !42, !c1_6) [-1, False] | |
| !s10 = StreamMap(!s9) { (%elt3) => | |
| !43 = GetTupleElement(%39) [0] | |
| !c1_7 = I32 [1] | |
| !44 = ApplyBinaryPrimOp(!43, !c1_7) [Subtract] | |
| !45 = ApplyComparisonOp(%elt3, !44) [EQWithNA] | |
| If !45 then { | |
| !46 = GetTupleElement(%39) [1] | |
| Cast(!46) [Int32] | |
| } else { | |
| ArrayRef(%22, %elt3) [-1] | |
| } | |
| } | |
| !s11 = StreamZip(!s8, !s10) [-1, AssumeSameLength] { | |
| (%elt4, %elt5) => | |
| MakeStruct(numberToTake: %elt5, old: %elt4) | |
| } | |
| ToArray(!s11) | |
| 2023-09-22 17:10:09.698 : INFO: after optimize: relationalLowerer, after LowerOrInterpretNonCompilable: IR size 127: | |
| !c4 = I32 [4] | |
| !c0 = I32 [0] | |
| %22 = TailLoop(!c4, !c0) { | |
| (%loopvar, %loopvar2, %loop) => | |
| !1 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s = ToStream(!1) [False] | |
| !s2 = StreamTake(!s, %loopvar) | |
| !2 = Literal [Struct{}, <literal value>] | |
| !3 = Str ["iteration="] | |
| !4 = Apply(%loopvar2) [-1, str, (), String] | |
| !5 = Apply(!3, !4) [-1, concat, (), String] | |
| !6 = Str [",nParts="] | |
| !7 = Apply(!5, !6) [-1, concat, (), String] | |
| !8 = Apply(%loopvar) [-1, str, (), String] | |
| !9 = Apply(!7, !8) [-1, concat, (), String] | |
| %12 = CollectDistributedArray(!s2, !2, !9) { | |
| (%ctx, %g) => | |
| !10 = GetField(%ctx) [start] | |
| !11 = GetField(%ctx) [end] | |
| !c1 = I32 [1] | |
| !s3 = StreamRange(!10, !11, !c1) [-1, True] | |
| !s4 = StreamMap(!s3) { (%elt) => | |
| MakeStruct(idx: %elt) | |
| } | |
| !c11 = I32 [11] | |
| !s5 = StreamTake(!s4, !c11) | |
| StreamLen(!s5) | |
| } | |
| !s6 = ToStream(%12) [False] | |
| !c0_2 = I32 [0] | |
| !13 = StreamFold(!s6, !c0_2) { (%accum, %elt2) => | |
| ApplyBinaryPrimOp(%accum, %elt2) [Add] | |
| } | |
| !14 = Cast(!13) [Int64] | |
| !15 = I64 [11] | |
| !16 = ApplyComparisonOp(!14, !15) [GTEQ] | |
| !c2 = I32 [2] | |
| !17 = ArrayLen(%12) | |
| !18 = ApplyComparisonOp(!c2, !17) [LTEQ] | |
| !19 = ApplySpecial(!16, !18) [-1, lor, | |
| (), Boolean] | |
| If !19 then { %12 } else { | |
| !c4_2 = I32 [4] | |
| !20 = ApplyBinaryPrimOp(%loopvar, !c4_2) [Multiply] | |
| !c1_2 = I32 [1] | |
| !21 = ApplyBinaryPrimOp(%loopvar2, !c1_2) [Add] | |
| Recur(!20, !21) [Array[Int32]] | |
| } | |
| } | |
| %23 = ArrayLen(%22) | |
| !c0_3 = I32 [0] | |
| !24 = ApplyComparisonOp(%23, !c0_3) [EQWithNA] | |
| %39 = If !24 then { | |
| Literal [Tuple[Int32,Int64], <literal value>] | |
| } else { | |
| !c0_4 = I32 [0] | |
| !25 = I64 [11] | |
| TailLoop(!c0_4, !25) { | |
| (%loopvar3, %loopvar4, %loop2) => | |
| !c1_3 = I32 [1] | |
| !26 = ApplyBinaryPrimOp(%23, !c1_3) [Subtract] | |
| !27 = ApplyComparisonOp(%loopvar3, !26) [EQWithNA] | |
| !28 = ArrayRef(%22, %loopvar3) [-1] | |
| !29 = Cast(!28) [Int64] | |
| !30 = ApplyBinaryPrimOp(%loopvar4, !29) [Subtract] | |
| !31 = I64 [0] | |
| !32 = ApplyComparisonOp(!30, !31) [LTEQ] | |
| !33 = ApplySpecial(!27, !32) [-1, lor, | |
| (), Boolean] | |
| If !33 then { | |
| !c1_4 = I32 [1] | |
| !34 = ApplyBinaryPrimOp(%loopvar3, !c1_4) [Add] | |
| MakeTuple(!34, %loopvar4) [(0 1)] | |
| } else { | |
| !c1_5 = I32 [1] | |
| !35 = ApplyBinaryPrimOp(%loopvar3, !c1_5) [Add] | |
| !36 = ArrayRef(%22, %loopvar3) [-1] | |
| !37 = Cast(!36) [Int64] | |
| !38 = ApplyBinaryPrimOp(%loopvar4, !37) [Subtract] | |
| Recur(!35, !38) [Tuple[Int32,Int64]] | |
| } | |
| } | |
| } | |
| !40 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s7 = ToStream(!40) [False] | |
| !41 = GetTupleElement(%39) [0] | |
| !s8 = StreamTake(!s7, !41) | |
| !c0_5 = I32 [0] | |
| !42 = GetTupleElement(%39) [0] | |
| !c1_6 = I32 [1] | |
| !s9 = StreamRange(!c0_5, !42, !c1_6) [-1, False] | |
| !s10 = StreamMap(!s9) { (%elt3) => | |
| !43 = GetTupleElement(%39) [0] | |
| !c1_7 = I32 [1] | |
| !44 = ApplyBinaryPrimOp(!43, !c1_7) [Subtract] | |
| !45 = ApplyComparisonOp(%elt3, !44) [EQWithNA] | |
| If !45 then { | |
| !46 = GetTupleElement(%39) [1] | |
| Cast(!46) [Int32] | |
| } else { | |
| ArrayRef(%22, %elt3) [-1] | |
| } | |
| } | |
| !s11 = StreamZip(!s8, !s10) [-1, AssumeSameLength] { | |
| (%elt4, %elt5) => | |
| MakeStruct(numberToTake: %elt5, old: %elt4) | |
| } | |
| ToArray(!s11) | |
| 2023-09-22 17:10:09.718 : INFO: initial IR: IR size 128: | |
| !c4 = I32 [4] | |
| !c0 = I32 [0] | |
| %22 = TailLoop(!c4, !c0) { | |
| (%loopvar, %loopvar2, %loop) => | |
| !1 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s = ToStream(!1) [False] | |
| !s2 = StreamTake(!s, %loopvar) | |
| !2 = Literal [Struct{}, <literal value>] | |
| !3 = Str ["iteration="] | |
| !4 = Apply(%loopvar2) [-1, str, (), String] | |
| !5 = Apply(!3, !4) [-1, concat, (), String] | |
| !6 = Str [",nParts="] | |
| !7 = Apply(!5, !6) [-1, concat, (), String] | |
| !8 = Apply(%loopvar) [-1, str, (), String] | |
| !9 = Apply(!7, !8) [-1, concat, (), String] | |
| %12 = CollectDistributedArray(!s2, !2, !9) { | |
| (%ctx, %g) => | |
| !10 = GetField(%ctx) [start] | |
| !11 = GetField(%ctx) [end] | |
| !c1 = I32 [1] | |
| !s3 = StreamRange(!10, !11, !c1) [-1, True] | |
| !s4 = StreamMap(!s3) { (%elt) => | |
| MakeStruct(idx: %elt) | |
| } | |
| !c11 = I32 [11] | |
| !s5 = StreamTake(!s4, !c11) | |
| StreamLen(!s5) | |
| } | |
| !s6 = ToStream(%12) [False] | |
| !c0_2 = I32 [0] | |
| !13 = StreamFold(!s6, !c0_2) { (%accum, %elt2) => | |
| ApplyBinaryPrimOp(%accum, %elt2) [Add] | |
| } | |
| !14 = Cast(!13) [Int64] | |
| !15 = I64 [11] | |
| !16 = ApplyComparisonOp(!14, !15) [GTEQ] | |
| !c2 = I32 [2] | |
| !17 = ArrayLen(%12) | |
| !18 = ApplyComparisonOp(!c2, !17) [LTEQ] | |
| !19 = ApplySpecial(!16, !18) [-1, lor, | |
| (), Boolean] | |
| If !19 then { %12 } else { | |
| !c4_2 = I32 [4] | |
| !20 = ApplyBinaryPrimOp(%loopvar, !c4_2) [Multiply] | |
| !c1_2 = I32 [1] | |
| !21 = ApplyBinaryPrimOp(%loopvar2, !c1_2) [Add] | |
| Recur(!20, !21) [Array[Int32]] | |
| } | |
| } | |
| %23 = ArrayLen(%22) | |
| !c0_3 = I32 [0] | |
| !24 = ApplyComparisonOp(%23, !c0_3) [EQWithNA] | |
| %39 = If !24 then { | |
| Literal [Tuple[Int32,Int64], <literal value>] | |
| } else { | |
| !c0_4 = I32 [0] | |
| !25 = I64 [11] | |
| TailLoop(!c0_4, !25) { | |
| (%loopvar3, %loopvar4, %loop2) => | |
| !c1_3 = I32 [1] | |
| !26 = ApplyBinaryPrimOp(%23, !c1_3) [Subtract] | |
| !27 = ApplyComparisonOp(%loopvar3, !26) [EQWithNA] | |
| !28 = ArrayRef(%22, %loopvar3) [-1] | |
| !29 = Cast(!28) [Int64] | |
| !30 = ApplyBinaryPrimOp(%loopvar4, !29) [Subtract] | |
| !31 = I64 [0] | |
| !32 = ApplyComparisonOp(!30, !31) [LTEQ] | |
| !33 = ApplySpecial(!27, !32) [-1, lor, | |
| (), Boolean] | |
| If !33 then { | |
| !c1_4 = I32 [1] | |
| !34 = ApplyBinaryPrimOp(%loopvar3, !c1_4) [Add] | |
| MakeTuple(!34, %loopvar4) [(0 1)] | |
| } else { | |
| !c1_5 = I32 [1] | |
| !35 = ApplyBinaryPrimOp(%loopvar3, !c1_5) [Add] | |
| !36 = ArrayRef(%22, %loopvar3) [-1] | |
| !37 = Cast(!36) [Int64] | |
| !38 = ApplyBinaryPrimOp(%loopvar4, !37) [Subtract] | |
| Recur(!35, !38) [Tuple[Int32,Int64]] | |
| } | |
| } | |
| } | |
| !40 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s7 = ToStream(!40) [False] | |
| !41 = GetTupleElement(%39) [0] | |
| !s8 = StreamTake(!s7, !41) | |
| !c0_5 = I32 [0] | |
| !42 = GetTupleElement(%39) [0] | |
| !c1_6 = I32 [1] | |
| !s9 = StreamRange(!c0_5, !42, !c1_6) [-1, False] | |
| !s10 = StreamMap(!s9) { (%elt3) => | |
| !43 = GetTupleElement(%39) [0] | |
| !c1_7 = I32 [1] | |
| !44 = ApplyBinaryPrimOp(!43, !c1_7) [Subtract] | |
| !45 = ApplyComparisonOp(%elt3, !44) [EQWithNA] | |
| If !45 then { | |
| !46 = GetTupleElement(%39) [1] | |
| Cast(!46) [Int32] | |
| } else { | |
| ArrayRef(%22, %elt3) [-1] | |
| } | |
| } | |
| !s11 = StreamZip(!s8, !s10) [-1, AssumeSameLength] { | |
| (%elt4, %elt5) => | |
| MakeStruct(numberToTake: %elt5, old: %elt4) | |
| } | |
| !47 = ToArray(!s11) | |
| MakeTuple(!47) [(0)] | |
| 2023-09-22 17:10:09.788 : INFO: after optimize: compileLowerer, initial IR: IR size 128: | |
| !c4 = I32 [4] | |
| !c0 = I32 [0] | |
| %22 = TailLoop(!c4, !c0) { | |
| (%loopvar, %loopvar2, %loop) => | |
| !1 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s = ToStream(!1) [False] | |
| !s2 = StreamTake(!s, %loopvar) | |
| !2 = Literal [Struct{}, <literal value>] | |
| !3 = Str ["iteration="] | |
| !4 = Apply(%loopvar2) [-1, str, (), String] | |
| !5 = Apply(!3, !4) [-1, concat, (), String] | |
| !6 = Str [",nParts="] | |
| !7 = Apply(!5, !6) [-1, concat, (), String] | |
| !8 = Apply(%loopvar) [-1, str, (), String] | |
| !9 = Apply(!7, !8) [-1, concat, (), String] | |
| %12 = CollectDistributedArray(!s2, !2, !9) { | |
| (%ctx, %g) => | |
| !10 = GetField(%ctx) [start] | |
| !11 = GetField(%ctx) [end] | |
| !c1 = I32 [1] | |
| !s3 = StreamRange(!10, !11, !c1) [-1, True] | |
| !s4 = StreamMap(!s3) { (%elt) => | |
| MakeStruct(idx: %elt) | |
| } | |
| !c11 = I32 [11] | |
| !s5 = StreamTake(!s4, !c11) | |
| StreamLen(!s5) | |
| } | |
| !s6 = ToStream(%12) [False] | |
| !c0_2 = I32 [0] | |
| !13 = StreamFold(!s6, !c0_2) { (%accum, %elt2) => | |
| ApplyBinaryPrimOp(%accum, %elt2) [Add] | |
| } | |
| !14 = Cast(!13) [Int64] | |
| !15 = I64 [11] | |
| !16 = ApplyComparisonOp(!14, !15) [GTEQ] | |
| !c2 = I32 [2] | |
| !17 = ArrayLen(%12) | |
| !18 = ApplyComparisonOp(!c2, !17) [LTEQ] | |
| !19 = ApplySpecial(!16, !18) [-1, lor, | |
| (), Boolean] | |
| If !19 then { %12 } else { | |
| !c4_2 = I32 [4] | |
| !20 = ApplyBinaryPrimOp(%loopvar, !c4_2) [Multiply] | |
| !c1_2 = I32 [1] | |
| !21 = ApplyBinaryPrimOp(%loopvar2, !c1_2) [Add] | |
| Recur(!20, !21) [Array[Int32]] | |
| } | |
| } | |
| %23 = ArrayLen(%22) | |
| !c0_3 = I32 [0] | |
| !24 = ApplyComparisonOp(%23, !c0_3) [EQWithNA] | |
| %39 = If !24 then { | |
| Literal [Tuple[Int32,Int64], <literal value>] | |
| } else { | |
| !c0_4 = I32 [0] | |
| !25 = I64 [11] | |
| TailLoop(!c0_4, !25) { | |
| (%loopvar3, %loopvar4, %loop2) => | |
| !c1_3 = I32 [1] | |
| !26 = ApplyBinaryPrimOp(%23, !c1_3) [Subtract] | |
| !27 = ApplyComparisonOp(%loopvar3, !26) [EQWithNA] | |
| !28 = ArrayRef(%22, %loopvar3) [-1] | |
| !29 = Cast(!28) [Int64] | |
| !30 = ApplyBinaryPrimOp(%loopvar4, !29) [Subtract] | |
| !31 = I64 [0] | |
| !32 = ApplyComparisonOp(!30, !31) [LTEQ] | |
| !33 = ApplySpecial(!27, !32) [-1, lor, | |
| (), Boolean] | |
| If !33 then { | |
| !c1_4 = I32 [1] | |
| !34 = ApplyBinaryPrimOp(%loopvar3, !c1_4) [Add] | |
| MakeTuple(!34, %loopvar4) [(0 1)] | |
| } else { | |
| !c1_5 = I32 [1] | |
| !35 = ApplyBinaryPrimOp(%loopvar3, !c1_5) [Add] | |
| !36 = ArrayRef(%22, %loopvar3) [-1] | |
| !37 = Cast(!36) [Int64] | |
| !38 = ApplyBinaryPrimOp(%loopvar4, !37) [Subtract] | |
| Recur(!35, !38) [Tuple[Int32,Int64]] | |
| } | |
| } | |
| } | |
| !40 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s7 = ToStream(!40) [False] | |
| !41 = GetTupleElement(%39) [0] | |
| !s8 = StreamTake(!s7, !41) | |
| !c0_5 = I32 [0] | |
| !42 = GetTupleElement(%39) [0] | |
| !c1_6 = I32 [1] | |
| !s9 = StreamRange(!c0_5, !42, !c1_6) [-1, False] | |
| !s10 = StreamMap(!s9) { (%elt3) => | |
| !43 = GetTupleElement(%39) [0] | |
| !c1_7 = I32 [1] | |
| !44 = ApplyBinaryPrimOp(!43, !c1_7) [Subtract] | |
| !45 = ApplyComparisonOp(%elt3, !44) [EQWithNA] | |
| If !45 then { | |
| !46 = GetTupleElement(%39) [1] | |
| Cast(!46) [Int32] | |
| } else { | |
| ArrayRef(%22, %elt3) [-1] | |
| } | |
| } | |
| !s11 = StreamZip(!s8, !s10) [-1, AssumeSameLength] { | |
| (%elt4, %elt5) => | |
| MakeStruct(numberToTake: %elt5, old: %elt4) | |
| } | |
| !47 = ToArray(!s11) | |
| MakeTuple(!47) [(0)] | |
| 2023-09-22 17:10:09.795 : INFO: after InlineApplyIR: IR size 128: | |
| !c4 = I32 [4] | |
| !c0 = I32 [0] | |
| %22 = TailLoop(!c4, !c0) { | |
| (%loopvar, %loopvar2, %loop) => | |
| !1 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s = ToStream(!1) [False] | |
| !s2 = StreamTake(!s, %loopvar) | |
| !2 = Literal [Struct{}, <literal value>] | |
| !3 = Str ["iteration="] | |
| !4 = Apply(%loopvar2) [-1, str, (), String] | |
| !5 = Apply(!3, !4) [-1, concat, (), String] | |
| !6 = Str [",nParts="] | |
| !7 = Apply(!5, !6) [-1, concat, (), String] | |
| !8 = Apply(%loopvar) [-1, str, (), String] | |
| !9 = Apply(!7, !8) [-1, concat, (), String] | |
| %12 = CollectDistributedArray(!s2, !2, !9) { | |
| (%ctx, %g) => | |
| !10 = GetField(%ctx) [start] | |
| !11 = GetField(%ctx) [end] | |
| !c1 = I32 [1] | |
| !s3 = StreamRange(!10, !11, !c1) [-1, True] | |
| !s4 = StreamMap(!s3) { (%elt) => | |
| MakeStruct(idx: %elt) | |
| } | |
| !c11 = I32 [11] | |
| !s5 = StreamTake(!s4, !c11) | |
| StreamLen(!s5) | |
| } | |
| !s6 = ToStream(%12) [False] | |
| !c0_2 = I32 [0] | |
| !13 = StreamFold(!s6, !c0_2) { (%accum, %elt2) => | |
| ApplyBinaryPrimOp(%accum, %elt2) [Add] | |
| } | |
| !14 = Cast(!13) [Int64] | |
| !15 = I64 [11] | |
| !16 = ApplyComparisonOp(!14, !15) [GTEQ] | |
| !c2 = I32 [2] | |
| !17 = ArrayLen(%12) | |
| !18 = ApplyComparisonOp(!c2, !17) [LTEQ] | |
| !19 = ApplySpecial(!16, !18) [-1, lor, | |
| (), Boolean] | |
| If !19 then { %12 } else { | |
| !c4_2 = I32 [4] | |
| !20 = ApplyBinaryPrimOp(%loopvar, !c4_2) [Multiply] | |
| !c1_2 = I32 [1] | |
| !21 = ApplyBinaryPrimOp(%loopvar2, !c1_2) [Add] | |
| Recur(!20, !21) [Array[Int32]] | |
| } | |
| } | |
| %23 = ArrayLen(%22) | |
| !c0_3 = I32 [0] | |
| !24 = ApplyComparisonOp(%23, !c0_3) [EQWithNA] | |
| %39 = If !24 then { | |
| Literal [Tuple[Int32,Int64], <literal value>] | |
| } else { | |
| !c0_4 = I32 [0] | |
| !25 = I64 [11] | |
| TailLoop(!c0_4, !25) { | |
| (%loopvar3, %loopvar4, %loop2) => | |
| !c1_3 = I32 [1] | |
| !26 = ApplyBinaryPrimOp(%23, !c1_3) [Subtract] | |
| !27 = ApplyComparisonOp(%loopvar3, !26) [EQWithNA] | |
| !28 = ArrayRef(%22, %loopvar3) [-1] | |
| !29 = Cast(!28) [Int64] | |
| !30 = ApplyBinaryPrimOp(%loopvar4, !29) [Subtract] | |
| !31 = I64 [0] | |
| !32 = ApplyComparisonOp(!30, !31) [LTEQ] | |
| !33 = ApplySpecial(!27, !32) [-1, lor, | |
| (), Boolean] | |
| If !33 then { | |
| !c1_4 = I32 [1] | |
| !34 = ApplyBinaryPrimOp(%loopvar3, !c1_4) [Add] | |
| MakeTuple(!34, %loopvar4) [(0 1)] | |
| } else { | |
| !c1_5 = I32 [1] | |
| !35 = ApplyBinaryPrimOp(%loopvar3, !c1_5) [Add] | |
| !36 = ArrayRef(%22, %loopvar3) [-1] | |
| !37 = Cast(!36) [Int64] | |
| !38 = ApplyBinaryPrimOp(%loopvar4, !37) [Subtract] | |
| Recur(!35, !38) [Tuple[Int32,Int64]] | |
| } | |
| } | |
| } | |
| !40 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s7 = ToStream(!40) [False] | |
| !41 = GetTupleElement(%39) [0] | |
| !s8 = StreamTake(!s7, !41) | |
| !c0_5 = I32 [0] | |
| !42 = GetTupleElement(%39) [0] | |
| !c1_6 = I32 [1] | |
| !s9 = StreamRange(!c0_5, !42, !c1_6) [-1, False] | |
| !s10 = StreamMap(!s9) { (%elt3) => | |
| !43 = GetTupleElement(%39) [0] | |
| !c1_7 = I32 [1] | |
| !44 = ApplyBinaryPrimOp(!43, !c1_7) [Subtract] | |
| !45 = ApplyComparisonOp(%elt3, !44) [EQWithNA] | |
| If !45 then { | |
| !46 = GetTupleElement(%39) [1] | |
| Cast(!46) [Int32] | |
| } else { | |
| ArrayRef(%22, %elt3) [-1] | |
| } | |
| } | |
| !s11 = StreamZip(!s8, !s10) [-1, AssumeSameLength] { | |
| (%elt4, %elt5) => | |
| MakeStruct(numberToTake: %elt5, old: %elt4) | |
| } | |
| !47 = ToArray(!s11) | |
| MakeTuple(!47) [(0)] | |
| 2023-09-22 17:10:09.832 : INFO: after optimize: compileLowerer, after InlineApplyIR: IR size 128: | |
| !c4 = I32 [4] | |
| !c0 = I32 [0] | |
| %22 = TailLoop(!c4, !c0) { | |
| (%loopvar, %loopvar2, %loop) => | |
| !1 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s = ToStream(!1) [False] | |
| !s2 = StreamTake(!s, %loopvar) | |
| !2 = Literal [Struct{}, <literal value>] | |
| !3 = Str ["iteration="] | |
| !4 = Apply(%loopvar2) [-1, str, (), String] | |
| !5 = Apply(!3, !4) [-1, concat, (), String] | |
| !6 = Str [",nParts="] | |
| !7 = Apply(!5, !6) [-1, concat, (), String] | |
| !8 = Apply(%loopvar) [-1, str, (), String] | |
| !9 = Apply(!7, !8) [-1, concat, (), String] | |
| %12 = CollectDistributedArray(!s2, !2, !9) { | |
| (%ctx, %g) => | |
| !10 = GetField(%ctx) [start] | |
| !11 = GetField(%ctx) [end] | |
| !c1 = I32 [1] | |
| !s3 = StreamRange(!10, !11, !c1) [-1, True] | |
| !s4 = StreamMap(!s3) { (%elt) => | |
| MakeStruct(idx: %elt) | |
| } | |
| !c11 = I32 [11] | |
| !s5 = StreamTake(!s4, !c11) | |
| StreamLen(!s5) | |
| } | |
| !s6 = ToStream(%12) [False] | |
| !c0_2 = I32 [0] | |
| !13 = StreamFold(!s6, !c0_2) { (%accum, %elt2) => | |
| ApplyBinaryPrimOp(%accum, %elt2) [Add] | |
| } | |
| !14 = Cast(!13) [Int64] | |
| !15 = I64 [11] | |
| !16 = ApplyComparisonOp(!14, !15) [GTEQ] | |
| !c2 = I32 [2] | |
| !17 = ArrayLen(%12) | |
| !18 = ApplyComparisonOp(!c2, !17) [LTEQ] | |
| !19 = ApplySpecial(!16, !18) [-1, lor, | |
| (), Boolean] | |
| If !19 then { %12 } else { | |
| !c4_2 = I32 [4] | |
| !20 = ApplyBinaryPrimOp(%loopvar, !c4_2) [Multiply] | |
| !c1_2 = I32 [1] | |
| !21 = ApplyBinaryPrimOp(%loopvar2, !c1_2) [Add] | |
| Recur(!20, !21) [Array[Int32]] | |
| } | |
| } | |
| %23 = ArrayLen(%22) | |
| !c0_3 = I32 [0] | |
| !24 = ApplyComparisonOp(%23, !c0_3) [EQWithNA] | |
| %39 = If !24 then { | |
| Literal [Tuple[Int32,Int64], <literal value>] | |
| } else { | |
| !c0_4 = I32 [0] | |
| !25 = I64 [11] | |
| TailLoop(!c0_4, !25) { | |
| (%loopvar3, %loopvar4, %loop2) => | |
| !c1_3 = I32 [1] | |
| !26 = ApplyBinaryPrimOp(%23, !c1_3) [Subtract] | |
| !27 = ApplyComparisonOp(%loopvar3, !26) [EQWithNA] | |
| !28 = ArrayRef(%22, %loopvar3) [-1] | |
| !29 = Cast(!28) [Int64] | |
| !30 = ApplyBinaryPrimOp(%loopvar4, !29) [Subtract] | |
| !31 = I64 [0] | |
| !32 = ApplyComparisonOp(!30, !31) [LTEQ] | |
| !33 = ApplySpecial(!27, !32) [-1, lor, | |
| (), Boolean] | |
| If !33 then { | |
| !c1_4 = I32 [1] | |
| !34 = ApplyBinaryPrimOp(%loopvar3, !c1_4) [Add] | |
| MakeTuple(!34, %loopvar4) [(0 1)] | |
| } else { | |
| !c1_5 = I32 [1] | |
| !35 = ApplyBinaryPrimOp(%loopvar3, !c1_5) [Add] | |
| !36 = ArrayRef(%22, %loopvar3) [-1] | |
| !37 = Cast(!36) [Int64] | |
| !38 = ApplyBinaryPrimOp(%loopvar4, !37) [Subtract] | |
| Recur(!35, !38) [Tuple[Int32,Int64]] | |
| } | |
| } | |
| } | |
| !40 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s7 = ToStream(!40) [False] | |
| !41 = GetTupleElement(%39) [0] | |
| !s8 = StreamTake(!s7, !41) | |
| !c0_5 = I32 [0] | |
| !42 = GetTupleElement(%39) [0] | |
| !c1_6 = I32 [1] | |
| !s9 = StreamRange(!c0_5, !42, !c1_6) [-1, False] | |
| !s10 = StreamMap(!s9) { (%elt3) => | |
| !43 = GetTupleElement(%39) [0] | |
| !c1_7 = I32 [1] | |
| !44 = ApplyBinaryPrimOp(!43, !c1_7) [Subtract] | |
| !45 = ApplyComparisonOp(%elt3, !44) [EQWithNA] | |
| If !45 then { | |
| !46 = GetTupleElement(%39) [1] | |
| Cast(!46) [Int32] | |
| } else { | |
| ArrayRef(%22, %elt3) [-1] | |
| } | |
| } | |
| !s11 = StreamZip(!s8, !s10) [-1, AssumeSameLength] { | |
| (%elt4, %elt5) => | |
| MakeStruct(numberToTake: %elt5, old: %elt4) | |
| } | |
| !47 = ToArray(!s11) | |
| MakeTuple(!47) [(0)] | |
| 2023-09-22 17:10:09.861 : INFO: after LowerArrayAggsToRunAggs: IR size 128: | |
| !c4 = I32 [4] | |
| !c0 = I32 [0] | |
| %22 = TailLoop(!c4, !c0) { | |
| (%loopvar, %loopvar2, %loop) => | |
| !1 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s = ToStream(!1) [False] | |
| !s2 = StreamTake(!s, %loopvar) | |
| !2 = Literal [Struct{}, <literal value>] | |
| !3 = Str ["iteration="] | |
| !4 = Apply(%loopvar2) [-1, str, (), String] | |
| !5 = Apply(!3, !4) [-1, concat, (), String] | |
| !6 = Str [",nParts="] | |
| !7 = Apply(!5, !6) [-1, concat, (), String] | |
| !8 = Apply(%loopvar) [-1, str, (), String] | |
| !9 = Apply(!7, !8) [-1, concat, (), String] | |
| %12 = CollectDistributedArray(!s2, !2, !9) { | |
| (%ctx, %g) => | |
| !10 = GetField(%ctx) [start] | |
| !11 = GetField(%ctx) [end] | |
| !c1 = I32 [1] | |
| !s3 = StreamRange(!10, !11, !c1) [-1, True] | |
| !s4 = StreamMap(!s3) { (%elt) => | |
| MakeStruct(idx: %elt) | |
| } | |
| !c11 = I32 [11] | |
| !s5 = StreamTake(!s4, !c11) | |
| StreamLen(!s5) | |
| } | |
| !s6 = ToStream(%12) [False] | |
| !c0_2 = I32 [0] | |
| !13 = StreamFold(!s6, !c0_2) { (%accum, %elt2) => | |
| ApplyBinaryPrimOp(%accum, %elt2) [Add] | |
| } | |
| !14 = Cast(!13) [Int64] | |
| !15 = I64 [11] | |
| !16 = ApplyComparisonOp(!14, !15) [GTEQ] | |
| !c2 = I32 [2] | |
| !17 = ArrayLen(%12) | |
| !18 = ApplyComparisonOp(!c2, !17) [LTEQ] | |
| !19 = ApplySpecial(!16, !18) [-1, lor, | |
| (), Boolean] | |
| If !19 then { %12 } else { | |
| !c4_2 = I32 [4] | |
| !20 = ApplyBinaryPrimOp(%loopvar, !c4_2) [Multiply] | |
| !c1_2 = I32 [1] | |
| !21 = ApplyBinaryPrimOp(%loopvar2, !c1_2) [Add] | |
| Recur(!20, !21) [Array[Int32]] | |
| } | |
| } | |
| %23 = ArrayLen(%22) | |
| !c0_3 = I32 [0] | |
| !24 = ApplyComparisonOp(%23, !c0_3) [EQWithNA] | |
| %39 = If !24 then { | |
| Literal [Tuple[Int32,Int64], <literal value>] | |
| } else { | |
| !c0_4 = I32 [0] | |
| !25 = I64 [11] | |
| TailLoop(!c0_4, !25) { | |
| (%loopvar3, %loopvar4, %loop2) => | |
| !c1_3 = I32 [1] | |
| !26 = ApplyBinaryPrimOp(%23, !c1_3) [Subtract] | |
| !27 = ApplyComparisonOp(%loopvar3, !26) [EQWithNA] | |
| !28 = ArrayRef(%22, %loopvar3) [-1] | |
| !29 = Cast(!28) [Int64] | |
| !30 = ApplyBinaryPrimOp(%loopvar4, !29) [Subtract] | |
| !31 = I64 [0] | |
| !32 = ApplyComparisonOp(!30, !31) [LTEQ] | |
| !33 = ApplySpecial(!27, !32) [-1, lor, | |
| (), Boolean] | |
| If !33 then { | |
| !c1_4 = I32 [1] | |
| !34 = ApplyBinaryPrimOp(%loopvar3, !c1_4) [Add] | |
| MakeTuple(!34, %loopvar4) [(0 1)] | |
| } else { | |
| !c1_5 = I32 [1] | |
| !35 = ApplyBinaryPrimOp(%loopvar3, !c1_5) [Add] | |
| !36 = ArrayRef(%22, %loopvar3) [-1] | |
| !37 = Cast(!36) [Int64] | |
| !38 = ApplyBinaryPrimOp(%loopvar4, !37) [Subtract] | |
| Recur(!35, !38) [Tuple[Int32,Int64]] | |
| } | |
| } | |
| } | |
| !40 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s7 = ToStream(!40) [False] | |
| !41 = GetTupleElement(%39) [0] | |
| !s8 = StreamTake(!s7, !41) | |
| !c0_5 = I32 [0] | |
| !42 = GetTupleElement(%39) [0] | |
| !c1_6 = I32 [1] | |
| !s9 = StreamRange(!c0_5, !42, !c1_6) [-1, False] | |
| !s10 = StreamMap(!s9) { (%elt3) => | |
| !43 = GetTupleElement(%39) [0] | |
| !c1_7 = I32 [1] | |
| !44 = ApplyBinaryPrimOp(!43, !c1_7) [Subtract] | |
| !45 = ApplyComparisonOp(%elt3, !44) [EQWithNA] | |
| If !45 then { | |
| !46 = GetTupleElement(%39) [1] | |
| Cast(!46) [Int32] | |
| } else { | |
| ArrayRef(%22, %elt3) [-1] | |
| } | |
| } | |
| !s11 = StreamZip(!s8, !s10) [-1, AssumeSameLength] { | |
| (%elt4, %elt5) => | |
| MakeStruct(numberToTake: %elt5, old: %elt4) | |
| } | |
| !47 = ToArray(!s11) | |
| MakeTuple(!47) [(0)] | |
| 2023-09-22 17:10:09.899 : INFO: after optimize: compileLowerer, after LowerArrayAggsToRunAggs: IR size 128: | |
| !c4 = I32 [4] | |
| !c0 = I32 [0] | |
| %22 = TailLoop(!c4, !c0) { | |
| (%loopvar, %loopvar2, %loop) => | |
| !1 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s = ToStream(!1) [False] | |
| !s2 = StreamTake(!s, %loopvar) | |
| !2 = Literal [Struct{}, <literal value>] | |
| !3 = Str ["iteration="] | |
| !4 = Apply(%loopvar2) [-1, str, (), String] | |
| !5 = Apply(!3, !4) [-1, concat, (), String] | |
| !6 = Str [",nParts="] | |
| !7 = Apply(!5, !6) [-1, concat, (), String] | |
| !8 = Apply(%loopvar) [-1, str, (), String] | |
| !9 = Apply(!7, !8) [-1, concat, (), String] | |
| %12 = CollectDistributedArray(!s2, !2, !9) { | |
| (%ctx, %g) => | |
| !10 = GetField(%ctx) [start] | |
| !11 = GetField(%ctx) [end] | |
| !c1 = I32 [1] | |
| !s3 = StreamRange(!10, !11, !c1) [-1, True] | |
| !s4 = StreamMap(!s3) { (%elt) => | |
| MakeStruct(idx: %elt) | |
| } | |
| !c11 = I32 [11] | |
| !s5 = StreamTake(!s4, !c11) | |
| StreamLen(!s5) | |
| } | |
| !s6 = ToStream(%12) [False] | |
| !c0_2 = I32 [0] | |
| !13 = StreamFold(!s6, !c0_2) { (%accum, %elt2) => | |
| ApplyBinaryPrimOp(%accum, %elt2) [Add] | |
| } | |
| !14 = Cast(!13) [Int64] | |
| !15 = I64 [11] | |
| !16 = ApplyComparisonOp(!14, !15) [GTEQ] | |
| !c2 = I32 [2] | |
| !17 = ArrayLen(%12) | |
| !18 = ApplyComparisonOp(!c2, !17) [LTEQ] | |
| !19 = ApplySpecial(!16, !18) [-1, lor, | |
| (), Boolean] | |
| If !19 then { %12 } else { | |
| !c4_2 = I32 [4] | |
| !20 = ApplyBinaryPrimOp(%loopvar, !c4_2) [Multiply] | |
| !c1_2 = I32 [1] | |
| !21 = ApplyBinaryPrimOp(%loopvar2, !c1_2) [Add] | |
| Recur(!20, !21) [Array[Int32]] | |
| } | |
| } | |
| %23 = ArrayLen(%22) | |
| !c0_3 = I32 [0] | |
| !24 = ApplyComparisonOp(%23, !c0_3) [EQWithNA] | |
| %39 = If !24 then { | |
| Literal [Tuple[Int32,Int64], <literal value>] | |
| } else { | |
| !c0_4 = I32 [0] | |
| !25 = I64 [11] | |
| TailLoop(!c0_4, !25) { | |
| (%loopvar3, %loopvar4, %loop2) => | |
| !c1_3 = I32 [1] | |
| !26 = ApplyBinaryPrimOp(%23, !c1_3) [Subtract] | |
| !27 = ApplyComparisonOp(%loopvar3, !26) [EQWithNA] | |
| !28 = ArrayRef(%22, %loopvar3) [-1] | |
| !29 = Cast(!28) [Int64] | |
| !30 = ApplyBinaryPrimOp(%loopvar4, !29) [Subtract] | |
| !31 = I64 [0] | |
| !32 = ApplyComparisonOp(!30, !31) [LTEQ] | |
| !33 = ApplySpecial(!27, !32) [-1, lor, | |
| (), Boolean] | |
| If !33 then { | |
| !c1_4 = I32 [1] | |
| !34 = ApplyBinaryPrimOp(%loopvar3, !c1_4) [Add] | |
| MakeTuple(!34, %loopvar4) [(0 1)] | |
| } else { | |
| !c1_5 = I32 [1] | |
| !35 = ApplyBinaryPrimOp(%loopvar3, !c1_5) [Add] | |
| !36 = ArrayRef(%22, %loopvar3) [-1] | |
| !37 = Cast(!36) [Int64] | |
| !38 = ApplyBinaryPrimOp(%loopvar4, !37) [Subtract] | |
| Recur(!35, !38) [Tuple[Int32,Int64]] | |
| } | |
| } | |
| } | |
| !40 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s7 = ToStream(!40) [False] | |
| !41 = GetTupleElement(%39) [0] | |
| !s8 = StreamTake(!s7, !41) | |
| !c0_5 = I32 [0] | |
| !42 = GetTupleElement(%39) [0] | |
| !c1_6 = I32 [1] | |
| !s9 = StreamRange(!c0_5, !42, !c1_6) [-1, False] | |
| !s10 = StreamMap(!s9) { (%elt3) => | |
| !43 = GetTupleElement(%39) [0] | |
| !c1_7 = I32 [1] | |
| !44 = ApplyBinaryPrimOp(!43, !c1_7) [Subtract] | |
| !45 = ApplyComparisonOp(%elt3, !44) [EQWithNA] | |
| If !45 then { | |
| !46 = GetTupleElement(%39) [1] | |
| Cast(!46) [Int32] | |
| } else { | |
| ArrayRef(%22, %elt3) [-1] | |
| } | |
| } | |
| !s11 = StreamZip(!s8, !s10) [-1, AssumeSameLength] { | |
| (%elt4, %elt5) => | |
| MakeStruct(numberToTake: %elt5, old: %elt4) | |
| } | |
| !47 = ToArray(!s11) | |
| MakeTuple(!47) [(0)] | |
| 2023-09-22 17:10:10.348 : INFO: instruction count: 3: __C22HailClassLoaderContainer.<init> | |
| 2023-09-22 17:10:10.348 : INFO: instruction count: 3: __C22HailClassLoaderContainer.<clinit> | |
| 2023-09-22 17:10:10.351 : INFO: instruction count: 3: __C24FSContainer.<init> | |
| 2023-09-22 17:10:10.352 : INFO: instruction count: 3: __C24FSContainer.<clinit> | |
| 2023-09-22 17:10:10.368 : INFO: instruction count: 3: __C26collect_distributed_array_table_head_recursive_count.<init> | |
| 2023-09-22 17:10:10.369 : INFO: instruction count: 111: __C26collect_distributed_array_table_head_recursive_count.apply | |
| 2023-09-22 17:10:10.369 : INFO: instruction count: 17: __C26collect_distributed_array_table_head_recursive_count.apply | |
| 2023-09-22 17:10:10.370 : INFO: instruction count: 27: __C26collect_distributed_array_table_head_recursive_count.__m28DECODE_r_struct_of_r_struct_of_r_int32ANDr_int32ENDEND_TO_SBaseStructPointer | |
| 2023-09-22 17:10:10.370 : INFO: instruction count: 26: __C26collect_distributed_array_table_head_recursive_count.__m29INPLACE_DECODE_r_struct_of_r_int32ANDr_int32END_TO_r_struct_of_r_int32ANDr_int32END | |
| 2023-09-22 17:10:10.370 : INFO: instruction count: 10: __C26collect_distributed_array_table_head_recursive_count.__m30INPLACE_DECODE_r_int32_TO_r_int32 | |
| 2023-09-22 17:10:10.371 : INFO: instruction count: 27: __C26collect_distributed_array_table_head_recursive_count.__m32DECODE_r_struct_of_r_struct_of_ENDEND_TO_SBaseStructPointer | |
| 2023-09-22 17:10:10.371 : INFO: instruction count: 8: __C26collect_distributed_array_table_head_recursive_count.__m33INPLACE_DECODE_r_struct_of_END_TO_r_struct_of_END | |
| 2023-09-22 17:10:10.371 YarnSchedulerBackend$YarnDriverEndpoint: INFO: Registered executor NettyRpcEndpointRef(spark-client://Executor) (10.240.0.5:57350) with ID 1, ResourceProfileId 0 | |
| 2023-09-22 17:10:10.374 : INFO: instruction count: 118: __C26collect_distributed_array_table_head_recursive_count.__m35split_StreamLen | |
| 2023-09-22 17:10:10.375 : INFO: instruction count: 13: __C26collect_distributed_array_table_head_recursive_count.__m45ENCODE_SBaseStructPointer_TO_r_struct_of_r_int32END | |
| 2023-09-22 17:10:10.375 : INFO: instruction count: 4: __C26collect_distributed_array_table_head_recursive_count.__m46ENCODE_SInt32$_TO_r_int32 | |
| 2023-09-22 17:10:10.375 : INFO: instruction count: 9: __C26collect_distributed_array_table_head_recursive_count.setPartitionIndex | |
| 2023-09-22 17:10:10.376 : INFO: instruction count: 4: __C26collect_distributed_array_table_head_recursive_count.addPartitionRegion | |
| 2023-09-22 17:10:10.376 : INFO: instruction count: 4: __C26collect_distributed_array_table_head_recursive_count.setPool | |
| 2023-09-22 17:10:10.376 : INFO: instruction count: 3: __C26collect_distributed_array_table_head_recursive_count.addHailClassLoader | |
| 2023-09-22 17:10:10.376 : INFO: instruction count: 3: __C26collect_distributed_array_table_head_recursive_count.addFS | |
| 2023-09-22 17:10:10.377 : INFO: instruction count: 4: __C26collect_distributed_array_table_head_recursive_count.addTaskContext | |
| 2023-09-22 17:10:10.377 ExecutorMonitor: INFO: New executor 1 has registered (new total is 1) | |
| 2023-09-22 17:10:10.463 BlockManagerMasterEndpoint: INFO: Registering block manager notebook-slowdown-repro-w-1.c.broad-ctsa.internal:45469 with 3.2 GiB RAM, BlockManagerId(1, notebook-slowdown-repro-w-1.c.broad-ctsa.internal, 45469, None) | |
| 2023-09-22 17:10:10.593 : INFO: encoder cache miss (0 hits, 1 misses, 0.000) | |
| 2023-09-22 17:10:10.608 : INFO: instruction count: 3: __C134HailClassLoaderContainer.<init> | |
| 2023-09-22 17:10:10.608 : INFO: instruction count: 3: __C134HailClassLoaderContainer.<clinit> | |
| 2023-09-22 17:10:10.609 : INFO: instruction count: 3: __C136FSContainer.<init> | |
| 2023-09-22 17:10:10.609 : INFO: instruction count: 3: __C136FSContainer.<clinit> | |
| 2023-09-22 17:10:10.614 : INFO: instruction count: 3: __C138etypeEncode.<init> | |
| 2023-09-22 17:10:10.614 : INFO: instruction count: 7: __C138etypeEncode.apply | |
| 2023-09-22 17:10:10.615 : INFO: instruction count: 65: __C138etypeEncode.__m140ENCODE_SBaseStructPointer_TO_r_struct_of_r_binaryANDr_binaryANDr_struct_of_ENDANDr_struct_of_r_int32ANDr_int64ENDANDr_array_of_r_struct_of_r_int32ANDr_int32ENDEND | |
| 2023-09-22 17:10:10.615 : INFO: instruction count: 16: __C138etypeEncode.__m141ENCODE_SStringPointer_TO_r_binary | |
| 2023-09-22 17:10:10.615 : INFO: instruction count: 1: __C138etypeEncode.__m142ENCODE_SBaseStructPointer_TO_r_struct_of_END | |
| 2023-09-22 17:10:10.616 : INFO: instruction count: 25: __C138etypeEncode.__m143ENCODE_SBaseStructPointer_TO_r_struct_of_r_int32ANDr_int64END | |
| 2023-09-22 17:10:10.616 : INFO: instruction count: 4: __C138etypeEncode.__m144ENCODE_SInt32$_TO_r_int32 | |
| 2023-09-22 17:10:10.616 : INFO: instruction count: 4: __C138etypeEncode.__m145ENCODE_SInt64$_TO_r_int64 | |
| 2023-09-22 17:10:10.617 : INFO: instruction count: 35: __C138etypeEncode.__m146ENCODE_SIndexablePointer_TO_r_array_of_r_struct_of_r_int32ANDr_int32END | |
| 2023-09-22 17:10:10.617 : INFO: instruction count: 25: __C138etypeEncode.__m147ENCODE_SBaseStructPointer_TO_r_struct_of_r_int32ANDr_int32END | |
| 2023-09-22 17:10:10.712 MemoryStore: INFO: Block broadcast_0 stored as values in memory (estimated size 136.0 B, free 24.4 GiB) | |
| 2023-09-22 17:10:11.162 MemoryStore: INFO: Block broadcast_0_piece0 stored as bytes in memory (estimated size 91.0 B, free 24.4 GiB) | |
| 2023-09-22 17:10:11.165 BlockManagerInfo: INFO: Added broadcast_0_piece0 in memory on notebook-slowdown-repro-m.c.broad-ctsa.internal:38207 (size: 91.0 B, free: 24.4 GiB) | |
| 2023-09-22 17:10:11.171 SparkContext: INFO: Created broadcast 0 from broadcast at SparkBackend.scala:361 | |
| 2023-09-22 17:10:11.173 : INFO: instruction count: 3: __C1HailClassLoaderContainer.<init> | |
| 2023-09-22 17:10:11.173 : INFO: instruction count: 3: __C1HailClassLoaderContainer.<clinit> | |
| 2023-09-22 17:10:11.174 : INFO: instruction count: 3: __C3FSContainer.<init> | |
| 2023-09-22 17:10:11.174 : INFO: instruction count: 3: __C3FSContainer.<clinit> | |
| 2023-09-22 17:10:11.231 : INFO: instruction count: 3: __C5Compiled.<init> | |
| 2023-09-22 17:10:11.232 : INFO: instruction count: 31: __C5Compiled.apply | |
| 2023-09-22 17:10:11.235 : INFO: instruction count: 431: __C5Compiled.__m7split_ToArray | |
| 2023-09-22 17:10:11.237 : INFO: instruction count: 371: __C5Compiled.__m9split_TailLoop | |
| 2023-09-22 17:10:11.237 : INFO: instruction count: 4: __C5Compiled.setBackend | |
| 2023-09-22 17:10:11.237 : INFO: instruction count: 9: __C5Compiled.__m55ENCODE_SBaseStructPointer_TO_r_struct_of_r_struct_of_r_int32ANDr_int32ENDEND | |
| 2023-09-22 17:10:11.238 : INFO: instruction count: 25: __C5Compiled.__m56ENCODE_SBaseStructPointer_TO_r_struct_of_r_int32ANDr_int32END | |
| 2023-09-22 17:10:11.238 : INFO: instruction count: 4: __C5Compiled.__m57ENCODE_SInt32$_TO_r_int32 | |
| 2023-09-22 17:10:11.238 : INFO: instruction count: 9: __C5Compiled.__m58ENCODE_SBaseStructPointer_TO_r_struct_of_r_struct_of_ENDEND | |
| 2023-09-22 17:10:11.238 : INFO: instruction count: 1: __C5Compiled.__m59ENCODE_SBaseStructPointer_TO_r_struct_of_END | |
| 2023-09-22 17:10:11.238 : INFO: instruction count: 18: __C5Compiled.__m60str | |
| 2023-09-22 17:10:11.239 : INFO: instruction count: 29: __C5Compiled.__m63concat | |
| 2023-09-22 17:10:11.239 : INFO: instruction count: 29: __C5Compiled.__m64concat | |
| 2023-09-22 17:10:11.239 : INFO: instruction count: 9: __C5Compiled.__m65concat | |
| 2023-09-22 17:10:11.239 : INFO: instruction count: 27: __C5Compiled.__m66DECODE_r_struct_of_r_int32END_TO_SBaseStructPointer | |
| 2023-09-22 17:10:11.240 : INFO: instruction count: 10: __C5Compiled.__m67INPLACE_DECODE_r_int32_TO_r_int32 | |
| 2023-09-22 17:10:11.240 : INFO: instruction count: 70: __C5Compiled.__m71split_StreamFold | |
| 2023-09-22 17:10:11.240 : INFO: instruction count: 11: __C5Compiled.__m81ord_gteq | |
| 2023-09-22 17:10:11.241 : INFO: instruction count: 16: __C5Compiled.__m82ord_gteqNonnull | |
| 2023-09-22 17:10:11.241 : INFO: instruction count: 11: __C5Compiled.__m83ord_lteq | |
| 2023-09-22 17:10:11.241 : INFO: instruction count: 14: __C5Compiled.__m84ord_lteqNonnull | |
| 2023-09-22 17:10:11.241 : INFO: instruction count: 11: __C5Compiled.__m92ord_equiv | |
| 2023-09-22 17:10:11.241 : INFO: instruction count: 14: __C5Compiled.__m93ord_equivNonnull | |
| 2023-09-22 17:10:11.242 : INFO: instruction count: 225: __C5Compiled.__m94split_TailLoop | |
| 2023-09-22 17:10:11.243 : INFO: instruction count: 35: __C5Compiled.__m100arrayref_bounds_check | |
| 2023-09-22 17:10:11.243 : INFO: instruction count: 11: __C5Compiled.__m101ord_lteq | |
| 2023-09-22 17:10:11.243 : INFO: instruction count: 16: __C5Compiled.__m102ord_lteqNonnull | |
| 2023-09-22 17:10:11.243 : INFO: instruction count: 9: __C5Compiled.setPartitionIndex | |
| 2023-09-22 17:10:11.243 : INFO: instruction count: 4: __C5Compiled.addPartitionRegion | |
| 2023-09-22 17:10:11.244 : INFO: instruction count: 4: __C5Compiled.setPool | |
| 2023-09-22 17:10:11.244 : INFO: instruction count: 3: __C5Compiled.addHailClassLoader | |
| 2023-09-22 17:10:11.244 : INFO: instruction count: 3: __C5Compiled.addFS | |
| 2023-09-22 17:10:11.244 : INFO: instruction count: 4: __C5Compiled.addTaskContext | |
| 2023-09-22 17:10:11.244 : INFO: instruction count: 3: __C5Compiled.setObjects | |
| 2023-09-22 17:10:11.247 : INFO: instruction count: 110: __C5Compiled.addAndDecodeLiterals | |
| 2023-09-22 17:10:11.247 : INFO: instruction count: 63: __C5Compiled.__m127DECODE_r_struct_of_r_binaryANDr_binaryANDr_struct_of_ENDANDr_struct_of_r_int32ANDr_int64ENDANDr_array_of_r_struct_of_r_int32ANDr_int32ENDEND_TO_SBaseStructPointer | |
| 2023-09-22 17:10:11.248 : INFO: instruction count: 31: __C5Compiled.__m128INPLACE_DECODE_r_binary_TO_r_string | |
| 2023-09-22 17:10:11.248 : INFO: instruction count: 8: __C5Compiled.__m129INPLACE_DECODE_r_struct_of_END_TO_r_struct_of_END | |
| 2023-09-22 17:10:11.248 : INFO: instruction count: 26: __C5Compiled.__m130INPLACE_DECODE_r_struct_of_r_int32ANDr_int64END_TO_r_tuple_of_r_int32ANDr_int64END | |
| 2023-09-22 17:10:11.248 : INFO: instruction count: 10: __C5Compiled.__m131INPLACE_DECODE_r_int64_TO_r_int64 | |
| 2023-09-22 17:10:11.249 : INFO: instruction count: 58: __C5Compiled.__m132INPLACE_DECODE_r_array_of_r_struct_of_r_int32ANDr_int32END_TO_r_array_of_r_struct_of_r_int32ANDr_int32END | |
| 2023-09-22 17:10:11.249 : INFO: instruction count: 26: __C5Compiled.__m133INPLACE_DECODE_r_struct_of_r_int32ANDr_int32END_TO_r_struct_of_r_int32ANDr_int32END | |
| 2023-09-22 17:10:11.250 : INFO: instruction count: 242: __C5Compiled.__m9split_TailLoop_region4_7 | |
| 2023-09-22 17:10:11.250 : INFO: instruction count: 3: __C148__m9split_TailLoopSpills.<init> | |
| 2023-09-22 17:10:11.251 : INFO: instruction count: 3: __C61staticWrapperClass_1.<init> | |
| 2023-09-22 17:10:11.254 : INFO: initial IR: IR size 128: | |
| !c4 = I32 [4] | |
| !c0 = I32 [0] | |
| %22 = TailLoop(!c4, !c0) { | |
| (%loopvar, %loopvar2, %loop) => | |
| !1 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s = ToStream(!1) [False] | |
| !s2 = StreamTake(!s, %loopvar) | |
| !2 = Literal [Struct{}, <literal value>] | |
| !3 = Str ["iteration="] | |
| !4 = Apply(%loopvar2) [-1, str, (), String] | |
| !5 = Apply(!3, !4) [-1, concat, (), String] | |
| !6 = Str [",nParts="] | |
| !7 = Apply(!5, !6) [-1, concat, (), String] | |
| !8 = Apply(%loopvar) [-1, str, (), String] | |
| !9 = Apply(!7, !8) [-1, concat, (), String] | |
| %12 = CollectDistributedArray(!s2, !2, !9) { | |
| (%ctx, %g) => | |
| !10 = GetField(%ctx) [start] | |
| !11 = GetField(%ctx) [end] | |
| !c1 = I32 [1] | |
| !s3 = StreamRange(!10, !11, !c1) [-1, True] | |
| !s4 = StreamMap(!s3) { (%elt) => | |
| MakeStruct(idx: %elt) | |
| } | |
| !c11 = I32 [11] | |
| !s5 = StreamTake(!s4, !c11) | |
| StreamLen(!s5) | |
| } | |
| !s6 = ToStream(%12) [False] | |
| !c0_2 = I32 [0] | |
| !13 = StreamFold(!s6, !c0_2) { (%accum, %elt2) => | |
| ApplyBinaryPrimOp(%accum, %elt2) [Add] | |
| } | |
| !14 = Cast(!13) [Int64] | |
| !15 = I64 [11] | |
| !16 = ApplyComparisonOp(!14, !15) [GTEQ] | |
| !c2 = I32 [2] | |
| !17 = ArrayLen(%12) | |
| !18 = ApplyComparisonOp(!c2, !17) [LTEQ] | |
| !19 = ApplySpecial(!16, !18) [-1, lor, | |
| (), Boolean] | |
| If !19 then { %12 } else { | |
| !c4_2 = I32 [4] | |
| !20 = ApplyBinaryPrimOp(%loopvar, !c4_2) [Multiply] | |
| !c1_2 = I32 [1] | |
| !21 = ApplyBinaryPrimOp(%loopvar2, !c1_2) [Add] | |
| Recur(!20, !21) [Array[Int32]] | |
| } | |
| } | |
| %23 = ArrayLen(%22) | |
| !c0_3 = I32 [0] | |
| !24 = ApplyComparisonOp(%23, !c0_3) [EQWithNA] | |
| %39 = If !24 then { | |
| Literal [Tuple[Int32,Int64], <literal value>] | |
| } else { | |
| !c0_4 = I32 [0] | |
| !25 = I64 [11] | |
| TailLoop(!c0_4, !25) { | |
| (%loopvar3, %loopvar4, %loop2) => | |
| !c1_3 = I32 [1] | |
| !26 = ApplyBinaryPrimOp(%23, !c1_3) [Subtract] | |
| !27 = ApplyComparisonOp(%loopvar3, !26) [EQWithNA] | |
| !28 = ArrayRef(%22, %loopvar3) [-1] | |
| !29 = Cast(!28) [Int64] | |
| !30 = ApplyBinaryPrimOp(%loopvar4, !29) [Subtract] | |
| !31 = I64 [0] | |
| !32 = ApplyComparisonOp(!30, !31) [LTEQ] | |
| !33 = ApplySpecial(!27, !32) [-1, lor, | |
| (), Boolean] | |
| If !33 then { | |
| !c1_4 = I32 [1] | |
| !34 = ApplyBinaryPrimOp(%loopvar3, !c1_4) [Add] | |
| MakeTuple(!34, %loopvar4) [(0 1)] | |
| } else { | |
| !c1_5 = I32 [1] | |
| !35 = ApplyBinaryPrimOp(%loopvar3, !c1_5) [Add] | |
| !36 = ArrayRef(%22, %loopvar3) [-1] | |
| !37 = Cast(!36) [Int64] | |
| !38 = ApplyBinaryPrimOp(%loopvar4, !37) [Subtract] | |
| Recur(!35, !38) [Tuple[Int32,Int64]] | |
| } | |
| } | |
| } | |
| !40 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s7 = ToStream(!40) [False] | |
| !41 = GetTupleElement(%39) [0] | |
| !s8 = StreamTake(!s7, !41) | |
| !c0_5 = I32 [0] | |
| !42 = GetTupleElement(%39) [0] | |
| !c1_6 = I32 [1] | |
| !s9 = StreamRange(!c0_5, !42, !c1_6) [-1, False] | |
| !s10 = StreamMap(!s9) { (%elt3) => | |
| !43 = GetTupleElement(%39) [0] | |
| !c1_7 = I32 [1] | |
| !44 = ApplyBinaryPrimOp(!43, !c1_7) [Subtract] | |
| !45 = ApplyComparisonOp(%elt3, !44) [EQWithNA] | |
| If !45 then { | |
| !46 = GetTupleElement(%39) [1] | |
| Cast(!46) [Int32] | |
| } else { | |
| ArrayRef(%22, %elt3) [-1] | |
| } | |
| } | |
| !s11 = StreamZip(!s8, !s10) [-1, AssumeSameLength] { | |
| (%elt4, %elt5) => | |
| MakeStruct(numberToTake: %elt5, old: %elt4) | |
| } | |
| !47 = ToArray(!s11) | |
| MakeTuple(!47) [(0)] | |
| 2023-09-22 17:10:11.286 : INFO: after optimize: compileLowerer, initial IR: IR size 128: | |
| !c4 = I32 [4] | |
| !c0 = I32 [0] | |
| %22 = TailLoop(!c4, !c0) { | |
| (%loopvar, %loopvar2, %loop) => | |
| !1 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s = ToStream(!1) [False] | |
| !s2 = StreamTake(!s, %loopvar) | |
| !2 = Literal [Struct{}, <literal value>] | |
| !3 = Str ["iteration="] | |
| !4 = Apply(%loopvar2) [-1, str, (), String] | |
| !5 = Apply(!3, !4) [-1, concat, (), String] | |
| !6 = Str [",nParts="] | |
| !7 = Apply(!5, !6) [-1, concat, (), String] | |
| !8 = Apply(%loopvar) [-1, str, (), String] | |
| !9 = Apply(!7, !8) [-1, concat, (), String] | |
| %12 = CollectDistributedArray(!s2, !2, !9) { | |
| (%ctx, %g) => | |
| !10 = GetField(%ctx) [start] | |
| !11 = GetField(%ctx) [end] | |
| !c1 = I32 [1] | |
| !s3 = StreamRange(!10, !11, !c1) [-1, True] | |
| !s4 = StreamMap(!s3) { (%elt) => | |
| MakeStruct(idx: %elt) | |
| } | |
| !c11 = I32 [11] | |
| !s5 = StreamTake(!s4, !c11) | |
| StreamLen(!s5) | |
| } | |
| !s6 = ToStream(%12) [False] | |
| !c0_2 = I32 [0] | |
| !13 = StreamFold(!s6, !c0_2) { (%accum, %elt2) => | |
| ApplyBinaryPrimOp(%accum, %elt2) [Add] | |
| } | |
| !14 = Cast(!13) [Int64] | |
| !15 = I64 [11] | |
| !16 = ApplyComparisonOp(!14, !15) [GTEQ] | |
| !c2 = I32 [2] | |
| !17 = ArrayLen(%12) | |
| !18 = ApplyComparisonOp(!c2, !17) [LTEQ] | |
| !19 = ApplySpecial(!16, !18) [-1, lor, | |
| (), Boolean] | |
| If !19 then { %12 } else { | |
| !c4_2 = I32 [4] | |
| !20 = ApplyBinaryPrimOp(%loopvar, !c4_2) [Multiply] | |
| !c1_2 = I32 [1] | |
| !21 = ApplyBinaryPrimOp(%loopvar2, !c1_2) [Add] | |
| Recur(!20, !21) [Array[Int32]] | |
| } | |
| } | |
| %23 = ArrayLen(%22) | |
| !c0_3 = I32 [0] | |
| !24 = ApplyComparisonOp(%23, !c0_3) [EQWithNA] | |
| %39 = If !24 then { | |
| Literal [Tuple[Int32,Int64], <literal value>] | |
| } else { | |
| !c0_4 = I32 [0] | |
| !25 = I64 [11] | |
| TailLoop(!c0_4, !25) { | |
| (%loopvar3, %loopvar4, %loop2) => | |
| !c1_3 = I32 [1] | |
| !26 = ApplyBinaryPrimOp(%23, !c1_3) [Subtract] | |
| !27 = ApplyComparisonOp(%loopvar3, !26) [EQWithNA] | |
| !28 = ArrayRef(%22, %loopvar3) [-1] | |
| !29 = Cast(!28) [Int64] | |
| !30 = ApplyBinaryPrimOp(%loopvar4, !29) [Subtract] | |
| !31 = I64 [0] | |
| !32 = ApplyComparisonOp(!30, !31) [LTEQ] | |
| !33 = ApplySpecial(!27, !32) [-1, lor, | |
| (), Boolean] | |
| If !33 then { | |
| !c1_4 = I32 [1] | |
| !34 = ApplyBinaryPrimOp(%loopvar3, !c1_4) [Add] | |
| MakeTuple(!34, %loopvar4) [(0 1)] | |
| } else { | |
| !c1_5 = I32 [1] | |
| !35 = ApplyBinaryPrimOp(%loopvar3, !c1_5) [Add] | |
| !36 = ArrayRef(%22, %loopvar3) [-1] | |
| !37 = Cast(!36) [Int64] | |
| !38 = ApplyBinaryPrimOp(%loopvar4, !37) [Subtract] | |
| Recur(!35, !38) [Tuple[Int32,Int64]] | |
| } | |
| } | |
| } | |
| !40 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s7 = ToStream(!40) [False] | |
| !41 = GetTupleElement(%39) [0] | |
| !s8 = StreamTake(!s7, !41) | |
| !c0_5 = I32 [0] | |
| !42 = GetTupleElement(%39) [0] | |
| !c1_6 = I32 [1] | |
| !s9 = StreamRange(!c0_5, !42, !c1_6) [-1, False] | |
| !s10 = StreamMap(!s9) { (%elt3) => | |
| !43 = GetTupleElement(%39) [0] | |
| !c1_7 = I32 [1] | |
| !44 = ApplyBinaryPrimOp(!43, !c1_7) [Subtract] | |
| !45 = ApplyComparisonOp(%elt3, !44) [EQWithNA] | |
| If !45 then { | |
| !46 = GetTupleElement(%39) [1] | |
| Cast(!46) [Int32] | |
| } else { | |
| ArrayRef(%22, %elt3) [-1] | |
| } | |
| } | |
| !s11 = StreamZip(!s8, !s10) [-1, AssumeSameLength] { | |
| (%elt4, %elt5) => | |
| MakeStruct(numberToTake: %elt5, old: %elt4) | |
| } | |
| !47 = ToArray(!s11) | |
| MakeTuple(!47) [(0)] | |
| 2023-09-22 17:10:11.291 : INFO: after InlineApplyIR: IR size 128: | |
| !c4 = I32 [4] | |
| !c0 = I32 [0] | |
| %22 = TailLoop(!c4, !c0) { | |
| (%loopvar, %loopvar2, %loop) => | |
| !1 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s = ToStream(!1) [False] | |
| !s2 = StreamTake(!s, %loopvar) | |
| !2 = Literal [Struct{}, <literal value>] | |
| !3 = Str ["iteration="] | |
| !4 = Apply(%loopvar2) [-1, str, (), String] | |
| !5 = Apply(!3, !4) [-1, concat, (), String] | |
| !6 = Str [",nParts="] | |
| !7 = Apply(!5, !6) [-1, concat, (), String] | |
| !8 = Apply(%loopvar) [-1, str, (), String] | |
| !9 = Apply(!7, !8) [-1, concat, (), String] | |
| %12 = CollectDistributedArray(!s2, !2, !9) { | |
| (%ctx, %g) => | |
| !10 = GetField(%ctx) [start] | |
| !11 = GetField(%ctx) [end] | |
| !c1 = I32 [1] | |
| !s3 = StreamRange(!10, !11, !c1) [-1, True] | |
| !s4 = StreamMap(!s3) { (%elt) => | |
| MakeStruct(idx: %elt) | |
| } | |
| !c11 = I32 [11] | |
| !s5 = StreamTake(!s4, !c11) | |
| StreamLen(!s5) | |
| } | |
| !s6 = ToStream(%12) [False] | |
| !c0_2 = I32 [0] | |
| !13 = StreamFold(!s6, !c0_2) { (%accum, %elt2) => | |
| ApplyBinaryPrimOp(%accum, %elt2) [Add] | |
| } | |
| !14 = Cast(!13) [Int64] | |
| !15 = I64 [11] | |
| !16 = ApplyComparisonOp(!14, !15) [GTEQ] | |
| !c2 = I32 [2] | |
| !17 = ArrayLen(%12) | |
| !18 = ApplyComparisonOp(!c2, !17) [LTEQ] | |
| !19 = ApplySpecial(!16, !18) [-1, lor, | |
| (), Boolean] | |
| If !19 then { %12 } else { | |
| !c4_2 = I32 [4] | |
| !20 = ApplyBinaryPrimOp(%loopvar, !c4_2) [Multiply] | |
| !c1_2 = I32 [1] | |
| !21 = ApplyBinaryPrimOp(%loopvar2, !c1_2) [Add] | |
| Recur(!20, !21) [Array[Int32]] | |
| } | |
| } | |
| %23 = ArrayLen(%22) | |
| !c0_3 = I32 [0] | |
| !24 = ApplyComparisonOp(%23, !c0_3) [EQWithNA] | |
| %39 = If !24 then { | |
| Literal [Tuple[Int32,Int64], <literal value>] | |
| } else { | |
| !c0_4 = I32 [0] | |
| !25 = I64 [11] | |
| TailLoop(!c0_4, !25) { | |
| (%loopvar3, %loopvar4, %loop2) => | |
| !c1_3 = I32 [1] | |
| !26 = ApplyBinaryPrimOp(%23, !c1_3) [Subtract] | |
| !27 = ApplyComparisonOp(%loopvar3, !26) [EQWithNA] | |
| !28 = ArrayRef(%22, %loopvar3) [-1] | |
| !29 = Cast(!28) [Int64] | |
| !30 = ApplyBinaryPrimOp(%loopvar4, !29) [Subtract] | |
| !31 = I64 [0] | |
| !32 = ApplyComparisonOp(!30, !31) [LTEQ] | |
| !33 = ApplySpecial(!27, !32) [-1, lor, | |
| (), Boolean] | |
| If !33 then { | |
| !c1_4 = I32 [1] | |
| !34 = ApplyBinaryPrimOp(%loopvar3, !c1_4) [Add] | |
| MakeTuple(!34, %loopvar4) [(0 1)] | |
| } else { | |
| !c1_5 = I32 [1] | |
| !35 = ApplyBinaryPrimOp(%loopvar3, !c1_5) [Add] | |
| !36 = ArrayRef(%22, %loopvar3) [-1] | |
| !37 = Cast(!36) [Int64] | |
| !38 = ApplyBinaryPrimOp(%loopvar4, !37) [Subtract] | |
| Recur(!35, !38) [Tuple[Int32,Int64]] | |
| } | |
| } | |
| } | |
| !40 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s7 = ToStream(!40) [False] | |
| !41 = GetTupleElement(%39) [0] | |
| !s8 = StreamTake(!s7, !41) | |
| !c0_5 = I32 [0] | |
| !42 = GetTupleElement(%39) [0] | |
| !c1_6 = I32 [1] | |
| !s9 = StreamRange(!c0_5, !42, !c1_6) [-1, False] | |
| !s10 = StreamMap(!s9) { (%elt3) => | |
| !43 = GetTupleElement(%39) [0] | |
| !c1_7 = I32 [1] | |
| !44 = ApplyBinaryPrimOp(!43, !c1_7) [Subtract] | |
| !45 = ApplyComparisonOp(%elt3, !44) [EQWithNA] | |
| If !45 then { | |
| !46 = GetTupleElement(%39) [1] | |
| Cast(!46) [Int32] | |
| } else { | |
| ArrayRef(%22, %elt3) [-1] | |
| } | |
| } | |
| !s11 = StreamZip(!s8, !s10) [-1, AssumeSameLength] { | |
| (%elt4, %elt5) => | |
| MakeStruct(numberToTake: %elt5, old: %elt4) | |
| } | |
| !47 = ToArray(!s11) | |
| MakeTuple(!47) [(0)] | |
| 2023-09-22 17:10:11.323 : INFO: after optimize: compileLowerer, after InlineApplyIR: IR size 128: | |
| !c4 = I32 [4] | |
| !c0 = I32 [0] | |
| %22 = TailLoop(!c4, !c0) { | |
| (%loopvar, %loopvar2, %loop) => | |
| !1 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s = ToStream(!1) [False] | |
| !s2 = StreamTake(!s, %loopvar) | |
| !2 = Literal [Struct{}, <literal value>] | |
| !3 = Str ["iteration="] | |
| !4 = Apply(%loopvar2) [-1, str, (), String] | |
| !5 = Apply(!3, !4) [-1, concat, (), String] | |
| !6 = Str [",nParts="] | |
| !7 = Apply(!5, !6) [-1, concat, (), String] | |
| !8 = Apply(%loopvar) [-1, str, (), String] | |
| !9 = Apply(!7, !8) [-1, concat, (), String] | |
| %12 = CollectDistributedArray(!s2, !2, !9) { | |
| (%ctx, %g) => | |
| !10 = GetField(%ctx) [start] | |
| !11 = GetField(%ctx) [end] | |
| !c1 = I32 [1] | |
| !s3 = StreamRange(!10, !11, !c1) [-1, True] | |
| !s4 = StreamMap(!s3) { (%elt) => | |
| MakeStruct(idx: %elt) | |
| } | |
| !c11 = I32 [11] | |
| !s5 = StreamTake(!s4, !c11) | |
| StreamLen(!s5) | |
| } | |
| !s6 = ToStream(%12) [False] | |
| !c0_2 = I32 [0] | |
| !13 = StreamFold(!s6, !c0_2) { (%accum, %elt2) => | |
| ApplyBinaryPrimOp(%accum, %elt2) [Add] | |
| } | |
| !14 = Cast(!13) [Int64] | |
| !15 = I64 [11] | |
| !16 = ApplyComparisonOp(!14, !15) [GTEQ] | |
| !c2 = I32 [2] | |
| !17 = ArrayLen(%12) | |
| !18 = ApplyComparisonOp(!c2, !17) [LTEQ] | |
| !19 = ApplySpecial(!16, !18) [-1, lor, | |
| (), Boolean] | |
| If !19 then { %12 } else { | |
| !c4_2 = I32 [4] | |
| !20 = ApplyBinaryPrimOp(%loopvar, !c4_2) [Multiply] | |
| !c1_2 = I32 [1] | |
| !21 = ApplyBinaryPrimOp(%loopvar2, !c1_2) [Add] | |
| Recur(!20, !21) [Array[Int32]] | |
| } | |
| } | |
| %23 = ArrayLen(%22) | |
| !c0_3 = I32 [0] | |
| !24 = ApplyComparisonOp(%23, !c0_3) [EQWithNA] | |
| %39 = If !24 then { | |
| Literal [Tuple[Int32,Int64], <literal value>] | |
| } else { | |
| !c0_4 = I32 [0] | |
| !25 = I64 [11] | |
| TailLoop(!c0_4, !25) { | |
| (%loopvar3, %loopvar4, %loop2) => | |
| !c1_3 = I32 [1] | |
| !26 = ApplyBinaryPrimOp(%23, !c1_3) [Subtract] | |
| !27 = ApplyComparisonOp(%loopvar3, !26) [EQWithNA] | |
| !28 = ArrayRef(%22, %loopvar3) [-1] | |
| !29 = Cast(!28) [Int64] | |
| !30 = ApplyBinaryPrimOp(%loopvar4, !29) [Subtract] | |
| !31 = I64 [0] | |
| !32 = ApplyComparisonOp(!30, !31) [LTEQ] | |
| !33 = ApplySpecial(!27, !32) [-1, lor, | |
| (), Boolean] | |
| If !33 then { | |
| !c1_4 = I32 [1] | |
| !34 = ApplyBinaryPrimOp(%loopvar3, !c1_4) [Add] | |
| MakeTuple(!34, %loopvar4) [(0 1)] | |
| } else { | |
| !c1_5 = I32 [1] | |
| !35 = ApplyBinaryPrimOp(%loopvar3, !c1_5) [Add] | |
| !36 = ArrayRef(%22, %loopvar3) [-1] | |
| !37 = Cast(!36) [Int64] | |
| !38 = ApplyBinaryPrimOp(%loopvar4, !37) [Subtract] | |
| Recur(!35, !38) [Tuple[Int32,Int64]] | |
| } | |
| } | |
| } | |
| !40 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s7 = ToStream(!40) [False] | |
| !41 = GetTupleElement(%39) [0] | |
| !s8 = StreamTake(!s7, !41) | |
| !c0_5 = I32 [0] | |
| !42 = GetTupleElement(%39) [0] | |
| !c1_6 = I32 [1] | |
| !s9 = StreamRange(!c0_5, !42, !c1_6) [-1, False] | |
| !s10 = StreamMap(!s9) { (%elt3) => | |
| !43 = GetTupleElement(%39) [0] | |
| !c1_7 = I32 [1] | |
| !44 = ApplyBinaryPrimOp(!43, !c1_7) [Subtract] | |
| !45 = ApplyComparisonOp(%elt3, !44) [EQWithNA] | |
| If !45 then { | |
| !46 = GetTupleElement(%39) [1] | |
| Cast(!46) [Int32] | |
| } else { | |
| ArrayRef(%22, %elt3) [-1] | |
| } | |
| } | |
| !s11 = StreamZip(!s8, !s10) [-1, AssumeSameLength] { | |
| (%elt4, %elt5) => | |
| MakeStruct(numberToTake: %elt5, old: %elt4) | |
| } | |
| !47 = ToArray(!s11) | |
| MakeTuple(!47) [(0)] | |
| 2023-09-22 17:10:11.332 : INFO: after LowerArrayAggsToRunAggs: IR size 128: | |
| !c4 = I32 [4] | |
| !c0 = I32 [0] | |
| %22 = TailLoop(!c4, !c0) { | |
| (%loopvar, %loopvar2, %loop) => | |
| !1 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s = ToStream(!1) [False] | |
| !s2 = StreamTake(!s, %loopvar) | |
| !2 = Literal [Struct{}, <literal value>] | |
| !3 = Str ["iteration="] | |
| !4 = Apply(%loopvar2) [-1, str, (), String] | |
| !5 = Apply(!3, !4) [-1, concat, (), String] | |
| !6 = Str [",nParts="] | |
| !7 = Apply(!5, !6) [-1, concat, (), String] | |
| !8 = Apply(%loopvar) [-1, str, (), String] | |
| !9 = Apply(!7, !8) [-1, concat, (), String] | |
| %12 = CollectDistributedArray(!s2, !2, !9) { | |
| (%ctx, %g) => | |
| !10 = GetField(%ctx) [start] | |
| !11 = GetField(%ctx) [end] | |
| !c1 = I32 [1] | |
| !s3 = StreamRange(!10, !11, !c1) [-1, True] | |
| !s4 = StreamMap(!s3) { (%elt) => | |
| MakeStruct(idx: %elt) | |
| } | |
| !c11 = I32 [11] | |
| !s5 = StreamTake(!s4, !c11) | |
| StreamLen(!s5) | |
| } | |
| !s6 = ToStream(%12) [False] | |
| !c0_2 = I32 [0] | |
| !13 = StreamFold(!s6, !c0_2) { (%accum, %elt2) => | |
| ApplyBinaryPrimOp(%accum, %elt2) [Add] | |
| } | |
| !14 = Cast(!13) [Int64] | |
| !15 = I64 [11] | |
| !16 = ApplyComparisonOp(!14, !15) [GTEQ] | |
| !c2 = I32 [2] | |
| !17 = ArrayLen(%12) | |
| !18 = ApplyComparisonOp(!c2, !17) [LTEQ] | |
| !19 = ApplySpecial(!16, !18) [-1, lor, | |
| (), Boolean] | |
| If !19 then { %12 } else { | |
| !c4_2 = I32 [4] | |
| !20 = ApplyBinaryPrimOp(%loopvar, !c4_2) [Multiply] | |
| !c1_2 = I32 [1] | |
| !21 = ApplyBinaryPrimOp(%loopvar2, !c1_2) [Add] | |
| Recur(!20, !21) [Array[Int32]] | |
| } | |
| } | |
| %23 = ArrayLen(%22) | |
| !c0_3 = I32 [0] | |
| !24 = ApplyComparisonOp(%23, !c0_3) [EQWithNA] | |
| %39 = If !24 then { | |
| Literal [Tuple[Int32,Int64], <literal value>] | |
| } else { | |
| !c0_4 = I32 [0] | |
| !25 = I64 [11] | |
| TailLoop(!c0_4, !25) { | |
| (%loopvar3, %loopvar4, %loop2) => | |
| !c1_3 = I32 [1] | |
| !26 = ApplyBinaryPrimOp(%23, !c1_3) [Subtract] | |
| !27 = ApplyComparisonOp(%loopvar3, !26) [EQWithNA] | |
| !28 = ArrayRef(%22, %loopvar3) [-1] | |
| !29 = Cast(!28) [Int64] | |
| !30 = ApplyBinaryPrimOp(%loopvar4, !29) [Subtract] | |
| !31 = I64 [0] | |
| !32 = ApplyComparisonOp(!30, !31) [LTEQ] | |
| !33 = ApplySpecial(!27, !32) [-1, lor, | |
| (), Boolean] | |
| If !33 then { | |
| !c1_4 = I32 [1] | |
| !34 = ApplyBinaryPrimOp(%loopvar3, !c1_4) [Add] | |
| MakeTuple(!34, %loopvar4) [(0 1)] | |
| } else { | |
| !c1_5 = I32 [1] | |
| !35 = ApplyBinaryPrimOp(%loopvar3, !c1_5) [Add] | |
| !36 = ArrayRef(%22, %loopvar3) [-1] | |
| !37 = Cast(!36) [Int64] | |
| !38 = ApplyBinaryPrimOp(%loopvar4, !37) [Subtract] | |
| Recur(!35, !38) [Tuple[Int32,Int64]] | |
| } | |
| } | |
| } | |
| !40 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s7 = ToStream(!40) [False] | |
| !41 = GetTupleElement(%39) [0] | |
| !s8 = StreamTake(!s7, !41) | |
| !c0_5 = I32 [0] | |
| !42 = GetTupleElement(%39) [0] | |
| !c1_6 = I32 [1] | |
| !s9 = StreamRange(!c0_5, !42, !c1_6) [-1, False] | |
| !s10 = StreamMap(!s9) { (%elt3) => | |
| !43 = GetTupleElement(%39) [0] | |
| !c1_7 = I32 [1] | |
| !44 = ApplyBinaryPrimOp(!43, !c1_7) [Subtract] | |
| !45 = ApplyComparisonOp(%elt3, !44) [EQWithNA] | |
| If !45 then { | |
| !46 = GetTupleElement(%39) [1] | |
| Cast(!46) [Int32] | |
| } else { | |
| ArrayRef(%22, %elt3) [-1] | |
| } | |
| } | |
| !s11 = StreamZip(!s8, !s10) [-1, AssumeSameLength] { | |
| (%elt4, %elt5) => | |
| MakeStruct(numberToTake: %elt5, old: %elt4) | |
| } | |
| !47 = ToArray(!s11) | |
| MakeTuple(!47) [(0)] | |
| 2023-09-22 17:10:11.363 : INFO: after optimize: compileLowerer, after LowerArrayAggsToRunAggs: IR size 128: | |
| !c4 = I32 [4] | |
| !c0 = I32 [0] | |
| %22 = TailLoop(!c4, !c0) { | |
| (%loopvar, %loopvar2, %loop) => | |
| !1 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s = ToStream(!1) [False] | |
| !s2 = StreamTake(!s, %loopvar) | |
| !2 = Literal [Struct{}, <literal value>] | |
| !3 = Str ["iteration="] | |
| !4 = Apply(%loopvar2) [-1, str, (), String] | |
| !5 = Apply(!3, !4) [-1, concat, (), String] | |
| !6 = Str [",nParts="] | |
| !7 = Apply(!5, !6) [-1, concat, (), String] | |
| !8 = Apply(%loopvar) [-1, str, (), String] | |
| !9 = Apply(!7, !8) [-1, concat, (), String] | |
| %12 = CollectDistributedArray(!s2, !2, !9) { | |
| (%ctx, %g) => | |
| !10 = GetField(%ctx) [start] | |
| !11 = GetField(%ctx) [end] | |
| !c1 = I32 [1] | |
| !s3 = StreamRange(!10, !11, !c1) [-1, True] | |
| !s4 = StreamMap(!s3) { (%elt) => | |
| MakeStruct(idx: %elt) | |
| } | |
| !c11 = I32 [11] | |
| !s5 = StreamTake(!s4, !c11) | |
| StreamLen(!s5) | |
| } | |
| !s6 = ToStream(%12) [False] | |
| !c0_2 = I32 [0] | |
| !13 = StreamFold(!s6, !c0_2) { (%accum, %elt2) => | |
| ApplyBinaryPrimOp(%accum, %elt2) [Add] | |
| } | |
| !14 = Cast(!13) [Int64] | |
| !15 = I64 [11] | |
| !16 = ApplyComparisonOp(!14, !15) [GTEQ] | |
| !c2 = I32 [2] | |
| !17 = ArrayLen(%12) | |
| !18 = ApplyComparisonOp(!c2, !17) [LTEQ] | |
| !19 = ApplySpecial(!16, !18) [-1, lor, | |
| (), Boolean] | |
| If !19 then { %12 } else { | |
| !c4_2 = I32 [4] | |
| !20 = ApplyBinaryPrimOp(%loopvar, !c4_2) [Multiply] | |
| !c1_2 = I32 [1] | |
| !21 = ApplyBinaryPrimOp(%loopvar2, !c1_2) [Add] | |
| Recur(!20, !21) [Array[Int32]] | |
| } | |
| } | |
| %23 = ArrayLen(%22) | |
| !c0_3 = I32 [0] | |
| !24 = ApplyComparisonOp(%23, !c0_3) [EQWithNA] | |
| %39 = If !24 then { | |
| Literal [Tuple[Int32,Int64], <literal value>] | |
| } else { | |
| !c0_4 = I32 [0] | |
| !25 = I64 [11] | |
| TailLoop(!c0_4, !25) { | |
| (%loopvar3, %loopvar4, %loop2) => | |
| !c1_3 = I32 [1] | |
| !26 = ApplyBinaryPrimOp(%23, !c1_3) [Subtract] | |
| !27 = ApplyComparisonOp(%loopvar3, !26) [EQWithNA] | |
| !28 = ArrayRef(%22, %loopvar3) [-1] | |
| !29 = Cast(!28) [Int64] | |
| !30 = ApplyBinaryPrimOp(%loopvar4, !29) [Subtract] | |
| !31 = I64 [0] | |
| !32 = ApplyComparisonOp(!30, !31) [LTEQ] | |
| !33 = ApplySpecial(!27, !32) [-1, lor, | |
| (), Boolean] | |
| If !33 then { | |
| !c1_4 = I32 [1] | |
| !34 = ApplyBinaryPrimOp(%loopvar3, !c1_4) [Add] | |
| MakeTuple(!34, %loopvar4) [(0 1)] | |
| } else { | |
| !c1_5 = I32 [1] | |
| !35 = ApplyBinaryPrimOp(%loopvar3, !c1_5) [Add] | |
| !36 = ArrayRef(%22, %loopvar3) [-1] | |
| !37 = Cast(!36) [Int64] | |
| !38 = ApplyBinaryPrimOp(%loopvar4, !37) [Subtract] | |
| Recur(!35, !38) [Tuple[Int32,Int64]] | |
| } | |
| } | |
| } | |
| !40 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s7 = ToStream(!40) [False] | |
| !41 = GetTupleElement(%39) [0] | |
| !s8 = StreamTake(!s7, !41) | |
| !c0_5 = I32 [0] | |
| !42 = GetTupleElement(%39) [0] | |
| !c1_6 = I32 [1] | |
| !s9 = StreamRange(!c0_5, !42, !c1_6) [-1, False] | |
| !s10 = StreamMap(!s9) { (%elt3) => | |
| !43 = GetTupleElement(%39) [0] | |
| !c1_7 = I32 [1] | |
| !44 = ApplyBinaryPrimOp(!43, !c1_7) [Subtract] | |
| !45 = ApplyComparisonOp(%elt3, !44) [EQWithNA] | |
| If !45 then { | |
| !46 = GetTupleElement(%39) [1] | |
| Cast(!46) [Int32] | |
| } else { | |
| ArrayRef(%22, %elt3) [-1] | |
| } | |
| } | |
| !s11 = StreamZip(!s8, !s10) [-1, AssumeSameLength] { | |
| (%elt4, %elt5) => | |
| MakeStruct(numberToTake: %elt5, old: %elt4) | |
| } | |
| !47 = ToArray(!s11) | |
| MakeTuple(!47) [(0)] | |
| 2023-09-22 17:10:11.387 : INFO: instruction count: 3: __C177HailClassLoaderContainer.<init> | |
| 2023-09-22 17:10:11.387 : INFO: instruction count: 3: __C177HailClassLoaderContainer.<clinit> | |
| 2023-09-22 17:10:11.388 : INFO: instruction count: 3: __C179FSContainer.<init> | |
| 2023-09-22 17:10:11.388 : INFO: instruction count: 3: __C179FSContainer.<clinit> | |
| 2023-09-22 17:10:11.392 : INFO: instruction count: 3: __C181collect_distributed_array_table_head_recursive_count.<init> | |
| 2023-09-22 17:10:11.392 : INFO: instruction count: 111: __C181collect_distributed_array_table_head_recursive_count.apply | |
| 2023-09-22 17:10:11.392 : INFO: instruction count: 17: __C181collect_distributed_array_table_head_recursive_count.apply | |
| 2023-09-22 17:10:11.393 : INFO: instruction count: 27: __C181collect_distributed_array_table_head_recursive_count.__m183DECODE_r_struct_of_r_struct_of_r_int32ANDr_int32ENDEND_TO_SBaseStructPointer | |
| 2023-09-22 17:10:11.393 : INFO: instruction count: 26: __C181collect_distributed_array_table_head_recursive_count.__m184INPLACE_DECODE_r_struct_of_r_int32ANDr_int32END_TO_r_struct_of_r_int32ANDr_int32END | |
| 2023-09-22 17:10:11.393 : INFO: instruction count: 10: __C181collect_distributed_array_table_head_recursive_count.__m185INPLACE_DECODE_r_int32_TO_r_int32 | |
| 2023-09-22 17:10:11.393 : INFO: instruction count: 27: __C181collect_distributed_array_table_head_recursive_count.__m187DECODE_r_struct_of_r_struct_of_ENDEND_TO_SBaseStructPointer | |
| 2023-09-22 17:10:11.393 : INFO: instruction count: 8: __C181collect_distributed_array_table_head_recursive_count.__m188INPLACE_DECODE_r_struct_of_END_TO_r_struct_of_END | |
| 2023-09-22 17:10:11.394 : INFO: instruction count: 118: __C181collect_distributed_array_table_head_recursive_count.__m190split_StreamLen | |
| 2023-09-22 17:10:11.394 : INFO: instruction count: 13: __C181collect_distributed_array_table_head_recursive_count.__m200ENCODE_SBaseStructPointer_TO_r_struct_of_r_int32END | |
| 2023-09-22 17:10:11.394 : INFO: instruction count: 4: __C181collect_distributed_array_table_head_recursive_count.__m201ENCODE_SInt32$_TO_r_int32 | |
| 2023-09-22 17:10:11.394 : INFO: instruction count: 9: __C181collect_distributed_array_table_head_recursive_count.setPartitionIndex | |
| 2023-09-22 17:10:11.395 : INFO: instruction count: 4: __C181collect_distributed_array_table_head_recursive_count.addPartitionRegion | |
| 2023-09-22 17:10:11.395 : INFO: instruction count: 4: __C181collect_distributed_array_table_head_recursive_count.setPool | |
| 2023-09-22 17:10:11.395 : INFO: instruction count: 3: __C181collect_distributed_array_table_head_recursive_count.addHailClassLoader | |
| 2023-09-22 17:10:11.395 : INFO: instruction count: 3: __C181collect_distributed_array_table_head_recursive_count.addFS | |
| 2023-09-22 17:10:11.395 : INFO: instruction count: 4: __C181collect_distributed_array_table_head_recursive_count.addTaskContext | |
| 2023-09-22 17:10:11.421 YarnSchedulerBackend$YarnDriverEndpoint: INFO: Registered executor NettyRpcEndpointRef(spark-client://Executor) (10.240.0.3:59140) with ID 2, ResourceProfileId 0 | |
| 2023-09-22 17:10:11.422 ExecutorMonitor: INFO: New executor 2 has registered (new total is 2) | |
| 2023-09-22 17:10:11.442 : INFO: encoder cache hit | |
| 2023-09-22 17:10:11.444 MemoryStore: INFO: Block broadcast_1 stored as values in memory (estimated size 136.0 B, free 24.4 GiB) | |
| 2023-09-22 17:10:11.449 MemoryStore: INFO: Block broadcast_1_piece0 stored as bytes in memory (estimated size 91.0 B, free 24.4 GiB) | |
| 2023-09-22 17:10:11.450 BlockManagerInfo: INFO: Added broadcast_1_piece0 in memory on notebook-slowdown-repro-m.c.broad-ctsa.internal:38207 (size: 91.0 B, free: 24.4 GiB) | |
| 2023-09-22 17:10:11.452 SparkContext: INFO: Created broadcast 1 from broadcast at SparkBackend.scala:361 | |
| 2023-09-22 17:10:11.453 : INFO: instruction count: 3: __C156HailClassLoaderContainer.<init> | |
| 2023-09-22 17:10:11.453 : INFO: instruction count: 3: __C156HailClassLoaderContainer.<clinit> | |
| 2023-09-22 17:10:11.454 : INFO: instruction count: 3: __C158FSContainer.<init> | |
| 2023-09-22 17:10:11.454 : INFO: instruction count: 3: __C158FSContainer.<clinit> | |
| 2023-09-22 17:10:11.482 : INFO: instruction count: 3: __C160Compiled.<init> | |
| 2023-09-22 17:10:11.483 : INFO: instruction count: 31: __C160Compiled.apply | |
| 2023-09-22 17:10:11.485 : INFO: instruction count: 431: __C160Compiled.__m162split_ToArray | |
| 2023-09-22 17:10:11.487 : INFO: instruction count: 371: __C160Compiled.__m164split_TailLoop | |
| 2023-09-22 17:10:11.488 : INFO: instruction count: 4: __C160Compiled.setBackend | |
| 2023-09-22 17:10:11.488 : INFO: instruction count: 9: __C160Compiled.__m210ENCODE_SBaseStructPointer_TO_r_struct_of_r_struct_of_r_int32ANDr_int32ENDEND | |
| 2023-09-22 17:10:11.488 : INFO: instruction count: 25: __C160Compiled.__m211ENCODE_SBaseStructPointer_TO_r_struct_of_r_int32ANDr_int32END | |
| 2023-09-22 17:10:11.488 : INFO: instruction count: 4: __C160Compiled.__m212ENCODE_SInt32$_TO_r_int32 | |
| 2023-09-22 17:10:11.489 : INFO: instruction count: 9: __C160Compiled.__m213ENCODE_SBaseStructPointer_TO_r_struct_of_r_struct_of_ENDEND | |
| 2023-09-22 17:10:11.489 : INFO: instruction count: 1: __C160Compiled.__m214ENCODE_SBaseStructPointer_TO_r_struct_of_END | |
| 2023-09-22 17:10:11.489 : INFO: instruction count: 18: __C160Compiled.__m215str | |
| 2023-09-22 17:10:11.489 : INFO: instruction count: 29: __C160Compiled.__m218concat | |
| 2023-09-22 17:10:11.490 : INFO: instruction count: 29: __C160Compiled.__m219concat | |
| 2023-09-22 17:10:11.490 : INFO: instruction count: 9: __C160Compiled.__m220concat | |
| 2023-09-22 17:10:11.490 : INFO: instruction count: 27: __C160Compiled.__m221DECODE_r_struct_of_r_int32END_TO_SBaseStructPointer | |
| 2023-09-22 17:10:11.491 : INFO: instruction count: 10: __C160Compiled.__m222INPLACE_DECODE_r_int32_TO_r_int32 | |
| 2023-09-22 17:10:11.491 : INFO: instruction count: 70: __C160Compiled.__m226split_StreamFold | |
| 2023-09-22 17:10:11.491 : INFO: instruction count: 11: __C160Compiled.__m236ord_gteq | |
| 2023-09-22 17:10:11.492 : INFO: instruction count: 16: __C160Compiled.__m237ord_gteqNonnull | |
| 2023-09-22 17:10:11.492 : INFO: instruction count: 11: __C160Compiled.__m238ord_lteq | |
| 2023-09-22 17:10:11.492 : INFO: instruction count: 14: __C160Compiled.__m239ord_lteqNonnull | |
| 2023-09-22 17:10:11.492 : INFO: instruction count: 11: __C160Compiled.__m247ord_equiv | |
| 2023-09-22 17:10:11.493 : INFO: instruction count: 14: __C160Compiled.__m248ord_equivNonnull | |
| 2023-09-22 17:10:11.494 : INFO: instruction count: 225: __C160Compiled.__m249split_TailLoop | |
| 2023-09-22 17:10:11.494 : INFO: instruction count: 35: __C160Compiled.__m255arrayref_bounds_check | |
| 2023-09-22 17:10:11.494 : INFO: instruction count: 11: __C160Compiled.__m256ord_lteq | |
| 2023-09-22 17:10:11.495 : INFO: instruction count: 16: __C160Compiled.__m257ord_lteqNonnull | |
| 2023-09-22 17:10:11.495 : INFO: instruction count: 9: __C160Compiled.setPartitionIndex | |
| 2023-09-22 17:10:11.495 : INFO: instruction count: 4: __C160Compiled.addPartitionRegion | |
| 2023-09-22 17:10:11.495 : INFO: instruction count: 4: __C160Compiled.setPool | |
| 2023-09-22 17:10:11.495 : INFO: instruction count: 3: __C160Compiled.addHailClassLoader | |
| 2023-09-22 17:10:11.496 : INFO: instruction count: 3: __C160Compiled.addFS | |
| 2023-09-22 17:10:11.496 : INFO: instruction count: 4: __C160Compiled.addTaskContext | |
| 2023-09-22 17:10:11.496 : INFO: instruction count: 3: __C160Compiled.setObjects | |
| 2023-09-22 17:10:11.496 : INFO: instruction count: 110: __C160Compiled.addAndDecodeLiterals | |
| 2023-09-22 17:10:11.497 : INFO: instruction count: 63: __C160Compiled.__m282DECODE_r_struct_of_r_binaryANDr_binaryANDr_struct_of_ENDANDr_struct_of_r_int32ANDr_int64ENDANDr_array_of_r_struct_of_r_int32ANDr_int32ENDEND_TO_SBaseStructPointer | |
| 2023-09-22 17:10:11.497 : INFO: instruction count: 31: __C160Compiled.__m283INPLACE_DECODE_r_binary_TO_r_string | |
| 2023-09-22 17:10:11.497 : INFO: instruction count: 8: __C160Compiled.__m284INPLACE_DECODE_r_struct_of_END_TO_r_struct_of_END | |
| 2023-09-22 17:10:11.498 : INFO: instruction count: 26: __C160Compiled.__m285INPLACE_DECODE_r_struct_of_r_int32ANDr_int64END_TO_r_tuple_of_r_int32ANDr_int64END | |
| 2023-09-22 17:10:11.498 : INFO: instruction count: 10: __C160Compiled.__m286INPLACE_DECODE_r_int64_TO_r_int64 | |
| 2023-09-22 17:10:11.498 : INFO: instruction count: 58: __C160Compiled.__m287INPLACE_DECODE_r_array_of_r_struct_of_r_int32ANDr_int32END_TO_r_array_of_r_struct_of_r_int32ANDr_int32END | |
| 2023-09-22 17:10:11.499 : INFO: instruction count: 26: __C160Compiled.__m288INPLACE_DECODE_r_struct_of_r_int32ANDr_int32END_TO_r_struct_of_r_int32ANDr_int32END | |
| 2023-09-22 17:10:11.500 : INFO: instruction count: 242: __C160Compiled.__m164split_TailLoop_region4_7 | |
| 2023-09-22 17:10:11.500 : INFO: instruction count: 3: __C289__m164split_TailLoopSpills.<init> | |
| 2023-09-22 17:10:11.501 : INFO: instruction count: 3: __C216staticWrapperClass_1.<init> | |
| 2023-09-22 17:10:11.506 : INFO: initial IR: IR size 128: | |
| !c4 = I32 [4] | |
| !c0 = I32 [0] | |
| %22 = TailLoop(!c4, !c0) { | |
| (%loopvar, %loopvar2, %loop) => | |
| !1 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s = ToStream(!1) [False] | |
| !s2 = StreamTake(!s, %loopvar) | |
| !2 = Literal [Struct{}, <literal value>] | |
| !3 = Str ["iteration="] | |
| !4 = Apply(%loopvar2) [-1, str, (), String] | |
| !5 = Apply(!3, !4) [-1, concat, (), String] | |
| !6 = Str [",nParts="] | |
| !7 = Apply(!5, !6) [-1, concat, (), String] | |
| !8 = Apply(%loopvar) [-1, str, (), String] | |
| !9 = Apply(!7, !8) [-1, concat, (), String] | |
| %12 = CollectDistributedArray(!s2, !2, !9) { | |
| (%ctx, %g) => | |
| !10 = GetField(%ctx) [start] | |
| !11 = GetField(%ctx) [end] | |
| !c1 = I32 [1] | |
| !s3 = StreamRange(!10, !11, !c1) [-1, True] | |
| !s4 = StreamMap(!s3) { (%elt) => | |
| MakeStruct(idx: %elt) | |
| } | |
| !c11 = I32 [11] | |
| !s5 = StreamTake(!s4, !c11) | |
| StreamLen(!s5) | |
| } | |
| !s6 = ToStream(%12) [False] | |
| !c0_2 = I32 [0] | |
| !13 = StreamFold(!s6, !c0_2) { (%accum, %elt2) => | |
| ApplyBinaryPrimOp(%accum, %elt2) [Add] | |
| } | |
| !14 = Cast(!13) [Int64] | |
| !15 = I64 [11] | |
| !16 = ApplyComparisonOp(!14, !15) [GTEQ] | |
| !c2 = I32 [2] | |
| !17 = ArrayLen(%12) | |
| !18 = ApplyComparisonOp(!c2, !17) [LTEQ] | |
| !19 = ApplySpecial(!16, !18) [-1, lor, | |
| (), Boolean] | |
| If !19 then { %12 } else { | |
| !c4_2 = I32 [4] | |
| !20 = ApplyBinaryPrimOp(%loopvar, !c4_2) [Multiply] | |
| !c1_2 = I32 [1] | |
| !21 = ApplyBinaryPrimOp(%loopvar2, !c1_2) [Add] | |
| Recur(!20, !21) [Array[Int32]] | |
| } | |
| } | |
| %23 = ArrayLen(%22) | |
| !c0_3 = I32 [0] | |
| !24 = ApplyComparisonOp(%23, !c0_3) [EQWithNA] | |
| %39 = If !24 then { | |
| Literal [Tuple[Int32,Int64], <literal value>] | |
| } else { | |
| !c0_4 = I32 [0] | |
| !25 = I64 [11] | |
| TailLoop(!c0_4, !25) { | |
| (%loopvar3, %loopvar4, %loop2) => | |
| !c1_3 = I32 [1] | |
| !26 = ApplyBinaryPrimOp(%23, !c1_3) [Subtract] | |
| !27 = ApplyComparisonOp(%loopvar3, !26) [EQWithNA] | |
| !28 = ArrayRef(%22, %loopvar3) [-1] | |
| !29 = Cast(!28) [Int64] | |
| !30 = ApplyBinaryPrimOp(%loopvar4, !29) [Subtract] | |
| !31 = I64 [0] | |
| !32 = ApplyComparisonOp(!30, !31) [LTEQ] | |
| !33 = ApplySpecial(!27, !32) [-1, lor, | |
| (), Boolean] | |
| If !33 then { | |
| !c1_4 = I32 [1] | |
| !34 = ApplyBinaryPrimOp(%loopvar3, !c1_4) [Add] | |
| MakeTuple(!34, %loopvar4) [(0 1)] | |
| } else { | |
| !c1_5 = I32 [1] | |
| !35 = ApplyBinaryPrimOp(%loopvar3, !c1_5) [Add] | |
| !36 = ArrayRef(%22, %loopvar3) [-1] | |
| !37 = Cast(!36) [Int64] | |
| !38 = ApplyBinaryPrimOp(%loopvar4, !37) [Subtract] | |
| Recur(!35, !38) [Tuple[Int32,Int64]] | |
| } | |
| } | |
| } | |
| !40 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s7 = ToStream(!40) [False] | |
| !41 = GetTupleElement(%39) [0] | |
| !s8 = StreamTake(!s7, !41) | |
| !c0_5 = I32 [0] | |
| !42 = GetTupleElement(%39) [0] | |
| !c1_6 = I32 [1] | |
| !s9 = StreamRange(!c0_5, !42, !c1_6) [-1, False] | |
| !s10 = StreamMap(!s9) { (%elt3) => | |
| !43 = GetTupleElement(%39) [0] | |
| !c1_7 = I32 [1] | |
| !44 = ApplyBinaryPrimOp(!43, !c1_7) [Subtract] | |
| !45 = ApplyComparisonOp(%elt3, !44) [EQWithNA] | |
| If !45 then { | |
| !46 = GetTupleElement(%39) [1] | |
| Cast(!46) [Int32] | |
| } else { | |
| ArrayRef(%22, %elt3) [-1] | |
| } | |
| } | |
| !s11 = StreamZip(!s8, !s10) [-1, AssumeSameLength] { | |
| (%elt4, %elt5) => | |
| MakeStruct(numberToTake: %elt5, old: %elt4) | |
| } | |
| !47 = ToArray(!s11) | |
| MakeTuple(!47) [(0)] | |
| 2023-09-22 17:10:11.515 BlockManagerMasterEndpoint: INFO: Registering block manager notebook-slowdown-repro-w-0.c.broad-ctsa.internal:38083 with 3.2 GiB RAM, BlockManagerId(2, notebook-slowdown-repro-w-0.c.broad-ctsa.internal, 38083, None) | |
| 2023-09-22 17:10:11.549 : INFO: after optimize: compileLowerer, initial IR: IR size 128: | |
| !c4 = I32 [4] | |
| !c0 = I32 [0] | |
| %22 = TailLoop(!c4, !c0) { | |
| (%loopvar, %loopvar2, %loop) => | |
| !1 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s = ToStream(!1) [False] | |
| !s2 = StreamTake(!s, %loopvar) | |
| !2 = Literal [Struct{}, <literal value>] | |
| !3 = Str ["iteration="] | |
| !4 = Apply(%loopvar2) [-1, str, (), String] | |
| !5 = Apply(!3, !4) [-1, concat, (), String] | |
| !6 = Str [",nParts="] | |
| !7 = Apply(!5, !6) [-1, concat, (), String] | |
| !8 = Apply(%loopvar) [-1, str, (), String] | |
| !9 = Apply(!7, !8) [-1, concat, (), String] | |
| %12 = CollectDistributedArray(!s2, !2, !9) { | |
| (%ctx, %g) => | |
| !10 = GetField(%ctx) [start] | |
| !11 = GetField(%ctx) [end] | |
| !c1 = I32 [1] | |
| !s3 = StreamRange(!10, !11, !c1) [-1, True] | |
| !s4 = StreamMap(!s3) { (%elt) => | |
| MakeStruct(idx: %elt) | |
| } | |
| !c11 = I32 [11] | |
| !s5 = StreamTake(!s4, !c11) | |
| StreamLen(!s5) | |
| } | |
| !s6 = ToStream(%12) [False] | |
| !c0_2 = I32 [0] | |
| !13 = StreamFold(!s6, !c0_2) { (%accum, %elt2) => | |
| ApplyBinaryPrimOp(%accum, %elt2) [Add] | |
| } | |
| !14 = Cast(!13) [Int64] | |
| !15 = I64 [11] | |
| !16 = ApplyComparisonOp(!14, !15) [GTEQ] | |
| !c2 = I32 [2] | |
| !17 = ArrayLen(%12) | |
| !18 = ApplyComparisonOp(!c2, !17) [LTEQ] | |
| !19 = ApplySpecial(!16, !18) [-1, lor, | |
| (), Boolean] | |
| If !19 then { %12 } else { | |
| !c4_2 = I32 [4] | |
| !20 = ApplyBinaryPrimOp(%loopvar, !c4_2) [Multiply] | |
| !c1_2 = I32 [1] | |
| !21 = ApplyBinaryPrimOp(%loopvar2, !c1_2) [Add] | |
| Recur(!20, !21) [Array[Int32]] | |
| } | |
| } | |
| %23 = ArrayLen(%22) | |
| !c0_3 = I32 [0] | |
| !24 = ApplyComparisonOp(%23, !c0_3) [EQWithNA] | |
| %39 = If !24 then { | |
| Literal [Tuple[Int32,Int64], <literal value>] | |
| } else { | |
| !c0_4 = I32 [0] | |
| !25 = I64 [11] | |
| TailLoop(!c0_4, !25) { | |
| (%loopvar3, %loopvar4, %loop2) => | |
| !c1_3 = I32 [1] | |
| !26 = ApplyBinaryPrimOp(%23, !c1_3) [Subtract] | |
| !27 = ApplyComparisonOp(%loopvar3, !26) [EQWithNA] | |
| !28 = ArrayRef(%22, %loopvar3) [-1] | |
| !29 = Cast(!28) [Int64] | |
| !30 = ApplyBinaryPrimOp(%loopvar4, !29) [Subtract] | |
| !31 = I64 [0] | |
| !32 = ApplyComparisonOp(!30, !31) [LTEQ] | |
| !33 = ApplySpecial(!27, !32) [-1, lor, | |
| (), Boolean] | |
| If !33 then { | |
| !c1_4 = I32 [1] | |
| !34 = ApplyBinaryPrimOp(%loopvar3, !c1_4) [Add] | |
| MakeTuple(!34, %loopvar4) [(0 1)] | |
| } else { | |
| !c1_5 = I32 [1] | |
| !35 = ApplyBinaryPrimOp(%loopvar3, !c1_5) [Add] | |
| !36 = ArrayRef(%22, %loopvar3) [-1] | |
| !37 = Cast(!36) [Int64] | |
| !38 = ApplyBinaryPrimOp(%loopvar4, !37) [Subtract] | |
| Recur(!35, !38) [Tuple[Int32,Int64]] | |
| } | |
| } | |
| } | |
| !40 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s7 = ToStream(!40) [False] | |
| !41 = GetTupleElement(%39) [0] | |
| !s8 = StreamTake(!s7, !41) | |
| !c0_5 = I32 [0] | |
| !42 = GetTupleElement(%39) [0] | |
| !c1_6 = I32 [1] | |
| !s9 = StreamRange(!c0_5, !42, !c1_6) [-1, False] | |
| !s10 = StreamMap(!s9) { (%elt3) => | |
| !43 = GetTupleElement(%39) [0] | |
| !c1_7 = I32 [1] | |
| !44 = ApplyBinaryPrimOp(!43, !c1_7) [Subtract] | |
| !45 = ApplyComparisonOp(%elt3, !44) [EQWithNA] | |
| If !45 then { | |
| !46 = GetTupleElement(%39) [1] | |
| Cast(!46) [Int32] | |
| } else { | |
| ArrayRef(%22, %elt3) [-1] | |
| } | |
| } | |
| !s11 = StreamZip(!s8, !s10) [-1, AssumeSameLength] { | |
| (%elt4, %elt5) => | |
| MakeStruct(numberToTake: %elt5, old: %elt4) | |
| } | |
| !47 = ToArray(!s11) | |
| MakeTuple(!47) [(0)] | |
| 2023-09-22 17:10:11.557 : INFO: after InlineApplyIR: IR size 128: | |
| !c4 = I32 [4] | |
| !c0 = I32 [0] | |
| %22 = TailLoop(!c4, !c0) { | |
| (%loopvar, %loopvar2, %loop) => | |
| !1 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s = ToStream(!1) [False] | |
| !s2 = StreamTake(!s, %loopvar) | |
| !2 = Literal [Struct{}, <literal value>] | |
| !3 = Str ["iteration="] | |
| !4 = Apply(%loopvar2) [-1, str, (), String] | |
| !5 = Apply(!3, !4) [-1, concat, (), String] | |
| !6 = Str [",nParts="] | |
| !7 = Apply(!5, !6) [-1, concat, (), String] | |
| !8 = Apply(%loopvar) [-1, str, (), String] | |
| !9 = Apply(!7, !8) [-1, concat, (), String] | |
| %12 = CollectDistributedArray(!s2, !2, !9) { | |
| (%ctx, %g) => | |
| !10 = GetField(%ctx) [start] | |
| !11 = GetField(%ctx) [end] | |
| !c1 = I32 [1] | |
| !s3 = StreamRange(!10, !11, !c1) [-1, True] | |
| !s4 = StreamMap(!s3) { (%elt) => | |
| MakeStruct(idx: %elt) | |
| } | |
| !c11 = I32 [11] | |
| !s5 = StreamTake(!s4, !c11) | |
| StreamLen(!s5) | |
| } | |
| !s6 = ToStream(%12) [False] | |
| !c0_2 = I32 [0] | |
| !13 = StreamFold(!s6, !c0_2) { (%accum, %elt2) => | |
| ApplyBinaryPrimOp(%accum, %elt2) [Add] | |
| } | |
| !14 = Cast(!13) [Int64] | |
| !15 = I64 [11] | |
| !16 = ApplyComparisonOp(!14, !15) [GTEQ] | |
| !c2 = I32 [2] | |
| !17 = ArrayLen(%12) | |
| !18 = ApplyComparisonOp(!c2, !17) [LTEQ] | |
| !19 = ApplySpecial(!16, !18) [-1, lor, | |
| (), Boolean] | |
| If !19 then { %12 } else { | |
| !c4_2 = I32 [4] | |
| !20 = ApplyBinaryPrimOp(%loopvar, !c4_2) [Multiply] | |
| !c1_2 = I32 [1] | |
| !21 = ApplyBinaryPrimOp(%loopvar2, !c1_2) [Add] | |
| Recur(!20, !21) [Array[Int32]] | |
| } | |
| } | |
| %23 = ArrayLen(%22) | |
| !c0_3 = I32 [0] | |
| !24 = ApplyComparisonOp(%23, !c0_3) [EQWithNA] | |
| %39 = If !24 then { | |
| Literal [Tuple[Int32,Int64], <literal value>] | |
| } else { | |
| !c0_4 = I32 [0] | |
| !25 = I64 [11] | |
| TailLoop(!c0_4, !25) { | |
| (%loopvar3, %loopvar4, %loop2) => | |
| !c1_3 = I32 [1] | |
| !26 = ApplyBinaryPrimOp(%23, !c1_3) [Subtract] | |
| !27 = ApplyComparisonOp(%loopvar3, !26) [EQWithNA] | |
| !28 = ArrayRef(%22, %loopvar3) [-1] | |
| !29 = Cast(!28) [Int64] | |
| !30 = ApplyBinaryPrimOp(%loopvar4, !29) [Subtract] | |
| !31 = I64 [0] | |
| !32 = ApplyComparisonOp(!30, !31) [LTEQ] | |
| !33 = ApplySpecial(!27, !32) [-1, lor, | |
| (), Boolean] | |
| If !33 then { | |
| !c1_4 = I32 [1] | |
| !34 = ApplyBinaryPrimOp(%loopvar3, !c1_4) [Add] | |
| MakeTuple(!34, %loopvar4) [(0 1)] | |
| } else { | |
| !c1_5 = I32 [1] | |
| !35 = ApplyBinaryPrimOp(%loopvar3, !c1_5) [Add] | |
| !36 = ArrayRef(%22, %loopvar3) [-1] | |
| !37 = Cast(!36) [Int64] | |
| !38 = ApplyBinaryPrimOp(%loopvar4, !37) [Subtract] | |
| Recur(!35, !38) [Tuple[Int32,Int64]] | |
| } | |
| } | |
| } | |
| !40 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s7 = ToStream(!40) [False] | |
| !41 = GetTupleElement(%39) [0] | |
| !s8 = StreamTake(!s7, !41) | |
| !c0_5 = I32 [0] | |
| !42 = GetTupleElement(%39) [0] | |
| !c1_6 = I32 [1] | |
| !s9 = StreamRange(!c0_5, !42, !c1_6) [-1, False] | |
| !s10 = StreamMap(!s9) { (%elt3) => | |
| !43 = GetTupleElement(%39) [0] | |
| !c1_7 = I32 [1] | |
| !44 = ApplyBinaryPrimOp(!43, !c1_7) [Subtract] | |
| !45 = ApplyComparisonOp(%elt3, !44) [EQWithNA] | |
| If !45 then { | |
| !46 = GetTupleElement(%39) [1] | |
| Cast(!46) [Int32] | |
| } else { | |
| ArrayRef(%22, %elt3) [-1] | |
| } | |
| } | |
| !s11 = StreamZip(!s8, !s10) [-1, AssumeSameLength] { | |
| (%elt4, %elt5) => | |
| MakeStruct(numberToTake: %elt5, old: %elt4) | |
| } | |
| !47 = ToArray(!s11) | |
| MakeTuple(!47) [(0)] | |
| 2023-09-22 17:10:11.595 : INFO: after optimize: compileLowerer, after InlineApplyIR: IR size 128: | |
| !c4 = I32 [4] | |
| !c0 = I32 [0] | |
| %22 = TailLoop(!c4, !c0) { | |
| (%loopvar, %loopvar2, %loop) => | |
| !1 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s = ToStream(!1) [False] | |
| !s2 = StreamTake(!s, %loopvar) | |
| !2 = Literal [Struct{}, <literal value>] | |
| !3 = Str ["iteration="] | |
| !4 = Apply(%loopvar2) [-1, str, (), String] | |
| !5 = Apply(!3, !4) [-1, concat, (), String] | |
| !6 = Str [",nParts="] | |
| !7 = Apply(!5, !6) [-1, concat, (), String] | |
| !8 = Apply(%loopvar) [-1, str, (), String] | |
| !9 = Apply(!7, !8) [-1, concat, (), String] | |
| %12 = CollectDistributedArray(!s2, !2, !9) { | |
| (%ctx, %g) => | |
| !10 = GetField(%ctx) [start] | |
| !11 = GetField(%ctx) [end] | |
| !c1 = I32 [1] | |
| !s3 = StreamRange(!10, !11, !c1) [-1, True] | |
| !s4 = StreamMap(!s3) { (%elt) => | |
| MakeStruct(idx: %elt) | |
| } | |
| !c11 = I32 [11] | |
| !s5 = StreamTake(!s4, !c11) | |
| StreamLen(!s5) | |
| } | |
| !s6 = ToStream(%12) [False] | |
| !c0_2 = I32 [0] | |
| !13 = StreamFold(!s6, !c0_2) { (%accum, %elt2) => | |
| ApplyBinaryPrimOp(%accum, %elt2) [Add] | |
| } | |
| !14 = Cast(!13) [Int64] | |
| !15 = I64 [11] | |
| !16 = ApplyComparisonOp(!14, !15) [GTEQ] | |
| !c2 = I32 [2] | |
| !17 = ArrayLen(%12) | |
| !18 = ApplyComparisonOp(!c2, !17) [LTEQ] | |
| !19 = ApplySpecial(!16, !18) [-1, lor, | |
| (), Boolean] | |
| If !19 then { %12 } else { | |
| !c4_2 = I32 [4] | |
| !20 = ApplyBinaryPrimOp(%loopvar, !c4_2) [Multiply] | |
| !c1_2 = I32 [1] | |
| !21 = ApplyBinaryPrimOp(%loopvar2, !c1_2) [Add] | |
| Recur(!20, !21) [Array[Int32]] | |
| } | |
| } | |
| %23 = ArrayLen(%22) | |
| !c0_3 = I32 [0] | |
| !24 = ApplyComparisonOp(%23, !c0_3) [EQWithNA] | |
| %39 = If !24 then { | |
| Literal [Tuple[Int32,Int64], <literal value>] | |
| } else { | |
| !c0_4 = I32 [0] | |
| !25 = I64 [11] | |
| TailLoop(!c0_4, !25) { | |
| (%loopvar3, %loopvar4, %loop2) => | |
| !c1_3 = I32 [1] | |
| !26 = ApplyBinaryPrimOp(%23, !c1_3) [Subtract] | |
| !27 = ApplyComparisonOp(%loopvar3, !26) [EQWithNA] | |
| !28 = ArrayRef(%22, %loopvar3) [-1] | |
| !29 = Cast(!28) [Int64] | |
| !30 = ApplyBinaryPrimOp(%loopvar4, !29) [Subtract] | |
| !31 = I64 [0] | |
| !32 = ApplyComparisonOp(!30, !31) [LTEQ] | |
| !33 = ApplySpecial(!27, !32) [-1, lor, | |
| (), Boolean] | |
| If !33 then { | |
| !c1_4 = I32 [1] | |
| !34 = ApplyBinaryPrimOp(%loopvar3, !c1_4) [Add] | |
| MakeTuple(!34, %loopvar4) [(0 1)] | |
| } else { | |
| !c1_5 = I32 [1] | |
| !35 = ApplyBinaryPrimOp(%loopvar3, !c1_5) [Add] | |
| !36 = ArrayRef(%22, %loopvar3) [-1] | |
| !37 = Cast(!36) [Int64] | |
| !38 = ApplyBinaryPrimOp(%loopvar4, !37) [Subtract] | |
| Recur(!35, !38) [Tuple[Int32,Int64]] | |
| } | |
| } | |
| } | |
| !40 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s7 = ToStream(!40) [False] | |
| !41 = GetTupleElement(%39) [0] | |
| !s8 = StreamTake(!s7, !41) | |
| !c0_5 = I32 [0] | |
| !42 = GetTupleElement(%39) [0] | |
| !c1_6 = I32 [1] | |
| !s9 = StreamRange(!c0_5, !42, !c1_6) [-1, False] | |
| !s10 = StreamMap(!s9) { (%elt3) => | |
| !43 = GetTupleElement(%39) [0] | |
| !c1_7 = I32 [1] | |
| !44 = ApplyBinaryPrimOp(!43, !c1_7) [Subtract] | |
| !45 = ApplyComparisonOp(%elt3, !44) [EQWithNA] | |
| If !45 then { | |
| !46 = GetTupleElement(%39) [1] | |
| Cast(!46) [Int32] | |
| } else { | |
| ArrayRef(%22, %elt3) [-1] | |
| } | |
| } | |
| !s11 = StreamZip(!s8, !s10) [-1, AssumeSameLength] { | |
| (%elt4, %elt5) => | |
| MakeStruct(numberToTake: %elt5, old: %elt4) | |
| } | |
| !47 = ToArray(!s11) | |
| MakeTuple(!47) [(0)] | |
| 2023-09-22 17:10:11.603 : INFO: after LowerArrayAggsToRunAggs: IR size 128: | |
| !c4 = I32 [4] | |
| !c0 = I32 [0] | |
| %22 = TailLoop(!c4, !c0) { | |
| (%loopvar, %loopvar2, %loop) => | |
| !1 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s = ToStream(!1) [False] | |
| !s2 = StreamTake(!s, %loopvar) | |
| !2 = Literal [Struct{}, <literal value>] | |
| !3 = Str ["iteration="] | |
| !4 = Apply(%loopvar2) [-1, str, (), String] | |
| !5 = Apply(!3, !4) [-1, concat, (), String] | |
| !6 = Str [",nParts="] | |
| !7 = Apply(!5, !6) [-1, concat, (), String] | |
| !8 = Apply(%loopvar) [-1, str, (), String] | |
| !9 = Apply(!7, !8) [-1, concat, (), String] | |
| %12 = CollectDistributedArray(!s2, !2, !9) { | |
| (%ctx, %g) => | |
| !10 = GetField(%ctx) [start] | |
| !11 = GetField(%ctx) [end] | |
| !c1 = I32 [1] | |
| !s3 = StreamRange(!10, !11, !c1) [-1, True] | |
| !s4 = StreamMap(!s3) { (%elt) => | |
| MakeStruct(idx: %elt) | |
| } | |
| !c11 = I32 [11] | |
| !s5 = StreamTake(!s4, !c11) | |
| StreamLen(!s5) | |
| } | |
| !s6 = ToStream(%12) [False] | |
| !c0_2 = I32 [0] | |
| !13 = StreamFold(!s6, !c0_2) { (%accum, %elt2) => | |
| ApplyBinaryPrimOp(%accum, %elt2) [Add] | |
| } | |
| !14 = Cast(!13) [Int64] | |
| !15 = I64 [11] | |
| !16 = ApplyComparisonOp(!14, !15) [GTEQ] | |
| !c2 = I32 [2] | |
| !17 = ArrayLen(%12) | |
| !18 = ApplyComparisonOp(!c2, !17) [LTEQ] | |
| !19 = ApplySpecial(!16, !18) [-1, lor, | |
| (), Boolean] | |
| If !19 then { %12 } else { | |
| !c4_2 = I32 [4] | |
| !20 = ApplyBinaryPrimOp(%loopvar, !c4_2) [Multiply] | |
| !c1_2 = I32 [1] | |
| !21 = ApplyBinaryPrimOp(%loopvar2, !c1_2) [Add] | |
| Recur(!20, !21) [Array[Int32]] | |
| } | |
| } | |
| %23 = ArrayLen(%22) | |
| !c0_3 = I32 [0] | |
| !24 = ApplyComparisonOp(%23, !c0_3) [EQWithNA] | |
| %39 = If !24 then { | |
| Literal [Tuple[Int32,Int64], <literal value>] | |
| } else { | |
| !c0_4 = I32 [0] | |
| !25 = I64 [11] | |
| TailLoop(!c0_4, !25) { | |
| (%loopvar3, %loopvar4, %loop2) => | |
| !c1_3 = I32 [1] | |
| !26 = ApplyBinaryPrimOp(%23, !c1_3) [Subtract] | |
| !27 = ApplyComparisonOp(%loopvar3, !26) [EQWithNA] | |
| !28 = ArrayRef(%22, %loopvar3) [-1] | |
| !29 = Cast(!28) [Int64] | |
| !30 = ApplyBinaryPrimOp(%loopvar4, !29) [Subtract] | |
| !31 = I64 [0] | |
| !32 = ApplyComparisonOp(!30, !31) [LTEQ] | |
| !33 = ApplySpecial(!27, !32) [-1, lor, | |
| (), Boolean] | |
| If !33 then { | |
| !c1_4 = I32 [1] | |
| !34 = ApplyBinaryPrimOp(%loopvar3, !c1_4) [Add] | |
| MakeTuple(!34, %loopvar4) [(0 1)] | |
| } else { | |
| !c1_5 = I32 [1] | |
| !35 = ApplyBinaryPrimOp(%loopvar3, !c1_5) [Add] | |
| !36 = ArrayRef(%22, %loopvar3) [-1] | |
| !37 = Cast(!36) [Int64] | |
| !38 = ApplyBinaryPrimOp(%loopvar4, !37) [Subtract] | |
| Recur(!35, !38) [Tuple[Int32,Int64]] | |
| } | |
| } | |
| } | |
| !40 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s7 = ToStream(!40) [False] | |
| !41 = GetTupleElement(%39) [0] | |
| !s8 = StreamTake(!s7, !41) | |
| !c0_5 = I32 [0] | |
| !42 = GetTupleElement(%39) [0] | |
| !c1_6 = I32 [1] | |
| !s9 = StreamRange(!c0_5, !42, !c1_6) [-1, False] | |
| !s10 = StreamMap(!s9) { (%elt3) => | |
| !43 = GetTupleElement(%39) [0] | |
| !c1_7 = I32 [1] | |
| !44 = ApplyBinaryPrimOp(!43, !c1_7) [Subtract] | |
| !45 = ApplyComparisonOp(%elt3, !44) [EQWithNA] | |
| If !45 then { | |
| !46 = GetTupleElement(%39) [1] | |
| Cast(!46) [Int32] | |
| } else { | |
| ArrayRef(%22, %elt3) [-1] | |
| } | |
| } | |
| !s11 = StreamZip(!s8, !s10) [-1, AssumeSameLength] { | |
| (%elt4, %elt5) => | |
| MakeStruct(numberToTake: %elt5, old: %elt4) | |
| } | |
| !47 = ToArray(!s11) | |
| MakeTuple(!47) [(0)] | |
| 2023-09-22 17:10:11.631 : INFO: after optimize: compileLowerer, after LowerArrayAggsToRunAggs: IR size 128: | |
| !c4 = I32 [4] | |
| !c0 = I32 [0] | |
| %22 = TailLoop(!c4, !c0) { | |
| (%loopvar, %loopvar2, %loop) => | |
| !1 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s = ToStream(!1) [False] | |
| !s2 = StreamTake(!s, %loopvar) | |
| !2 = Literal [Struct{}, <literal value>] | |
| !3 = Str ["iteration="] | |
| !4 = Apply(%loopvar2) [-1, str, (), String] | |
| !5 = Apply(!3, !4) [-1, concat, (), String] | |
| !6 = Str [",nParts="] | |
| !7 = Apply(!5, !6) [-1, concat, (), String] | |
| !8 = Apply(%loopvar) [-1, str, (), String] | |
| !9 = Apply(!7, !8) [-1, concat, (), String] | |
| %12 = CollectDistributedArray(!s2, !2, !9) { | |
| (%ctx, %g) => | |
| !10 = GetField(%ctx) [start] | |
| !11 = GetField(%ctx) [end] | |
| !c1 = I32 [1] | |
| !s3 = StreamRange(!10, !11, !c1) [-1, True] | |
| !s4 = StreamMap(!s3) { (%elt) => | |
| MakeStruct(idx: %elt) | |
| } | |
| !c11 = I32 [11] | |
| !s5 = StreamTake(!s4, !c11) | |
| StreamLen(!s5) | |
| } | |
| !s6 = ToStream(%12) [False] | |
| !c0_2 = I32 [0] | |
| !13 = StreamFold(!s6, !c0_2) { (%accum, %elt2) => | |
| ApplyBinaryPrimOp(%accum, %elt2) [Add] | |
| } | |
| !14 = Cast(!13) [Int64] | |
| !15 = I64 [11] | |
| !16 = ApplyComparisonOp(!14, !15) [GTEQ] | |
| !c2 = I32 [2] | |
| !17 = ArrayLen(%12) | |
| !18 = ApplyComparisonOp(!c2, !17) [LTEQ] | |
| !19 = ApplySpecial(!16, !18) [-1, lor, | |
| (), Boolean] | |
| If !19 then { %12 } else { | |
| !c4_2 = I32 [4] | |
| !20 = ApplyBinaryPrimOp(%loopvar, !c4_2) [Multiply] | |
| !c1_2 = I32 [1] | |
| !21 = ApplyBinaryPrimOp(%loopvar2, !c1_2) [Add] | |
| Recur(!20, !21) [Array[Int32]] | |
| } | |
| } | |
| %23 = ArrayLen(%22) | |
| !c0_3 = I32 [0] | |
| !24 = ApplyComparisonOp(%23, !c0_3) [EQWithNA] | |
| %39 = If !24 then { | |
| Literal [Tuple[Int32,Int64], <literal value>] | |
| } else { | |
| !c0_4 = I32 [0] | |
| !25 = I64 [11] | |
| TailLoop(!c0_4, !25) { | |
| (%loopvar3, %loopvar4, %loop2) => | |
| !c1_3 = I32 [1] | |
| !26 = ApplyBinaryPrimOp(%23, !c1_3) [Subtract] | |
| !27 = ApplyComparisonOp(%loopvar3, !26) [EQWithNA] | |
| !28 = ArrayRef(%22, %loopvar3) [-1] | |
| !29 = Cast(!28) [Int64] | |
| !30 = ApplyBinaryPrimOp(%loopvar4, !29) [Subtract] | |
| !31 = I64 [0] | |
| !32 = ApplyComparisonOp(!30, !31) [LTEQ] | |
| !33 = ApplySpecial(!27, !32) [-1, lor, | |
| (), Boolean] | |
| If !33 then { | |
| !c1_4 = I32 [1] | |
| !34 = ApplyBinaryPrimOp(%loopvar3, !c1_4) [Add] | |
| MakeTuple(!34, %loopvar4) [(0 1)] | |
| } else { | |
| !c1_5 = I32 [1] | |
| !35 = ApplyBinaryPrimOp(%loopvar3, !c1_5) [Add] | |
| !36 = ArrayRef(%22, %loopvar3) [-1] | |
| !37 = Cast(!36) [Int64] | |
| !38 = ApplyBinaryPrimOp(%loopvar4, !37) [Subtract] | |
| Recur(!35, !38) [Tuple[Int32,Int64]] | |
| } | |
| } | |
| } | |
| !40 = Literal [Array[Struct{start:Int32,end:Int32}], <literal value>] | |
| !s7 = ToStream(!40) [False] | |
| !41 = GetTupleElement(%39) [0] | |
| !s8 = StreamTake(!s7, !41) | |
| !c0_5 = I32 [0] | |
| !42 = GetTupleElement(%39) [0] | |
| !c1_6 = I32 [1] | |
| !s9 = StreamRange(!c0_5, !42, !c1_6) [-1, False] | |
| !s10 = StreamMap(!s9) { (%elt3) => | |
| !43 = GetTupleElement(%39) [0] | |
| !c1_7 = I32 [1] | |
| !44 = ApplyBinaryPrimOp(!43, !c1_7) [Subtract] | |
| !45 = ApplyComparisonOp(%elt3, !44) [EQWithNA] | |
| If !45 then { | |
| !46 = GetTupleElement(%39) [1] | |
| Cast(!46) [Int32] | |
| } else { | |
| ArrayRef(%22, %elt3) [-1] | |
| } | |
| } | |
| !s11 = StreamZip(!s8, !s10) [-1, AssumeSameLength] { | |
| (%elt4, %elt5) => | |
| MakeStruct(numberToTake: %elt5, old: %elt4) | |
| } | |
| !47 = ToArray(!s11) | |
| MakeTuple(!47) [(0)] | |
| 2023-09-22 17:10:11.648 : INFO: instruction count: 3: __C318HailClassLoaderContainer.<init> | |
| 2023-09-22 17:10:11.648 : INFO: instruction count: 3: __C318HailClassLoaderContainer.<clinit> | |
| 2023-09-22 17:10:11.649 : INFO: instruction count: 3: __C320FSContainer.<init> | |
| 2023-09-22 17:10:11.649 : INFO: instruction count: 3: __C320FSContainer.<clinit> | |
| 2023-09-22 17:10:11.652 : INFO: instruction count: 3: __C322collect_distributed_array_table_head_recursive_count.<init> | |
| 2023-09-22 17:10:11.653 : INFO: instruction count: 111: __C322collect_distributed_array_table_head_recursive_count.apply | |
| 2023-09-22 17:10:11.653 : INFO: instruction count: 17: __C322collect_distributed_array_table_head_recursive_count.apply | |
| 2023-09-22 17:10:11.653 : INFO: instruction count: 27: __C322collect_distributed_array_table_head_recursive_count.__m324DECODE_r_struct_of_r_struct_of_r_int32ANDr_int32ENDEND_TO_SBaseStructPointer | |
| 2023-09-22 17:10:11.653 : INFO: instruction count: 26: __C322collect_distributed_array_table_head_recursive_count.__m325INPLACE_DECODE_r_struct_of_r_int32ANDr_int32END_TO_r_struct_of_r_int32ANDr_int32END | |
| 2023-09-22 17:10:11.653 : INFO: instruction count: 10: __C322collect_distributed_array_table_head_recursive_count.__m326INPLACE_DECODE_r_int32_TO_r_int32 | |
| 2023-09-22 17:10:11.654 : INFO: instruction count: 27: __C322collect_distributed_array_table_head_recursive_count.__m328DECODE_r_struct_of_r_struct_of_ENDEND_TO_SBaseStructPointer | |
| 2023-09-22 17:10:11.654 : INFO: instruction count: 8: __C322collect_distributed_array_table_head_recursive_count.__m329INPLACE_DECODE_r_struct_of_END_TO_r_struct_of_END | |
| 2023-09-22 17:10:11.654 : INFO: instruction count: 118: __C322collect_distributed_array_table_head_recursive_count.__m331split_StreamLen | |
| 2023-09-22 17:10:11.654 : INFO: instruction count: 13: __C322collect_distributed_array_table_head_recursive_count.__m341ENCODE_SBaseStructPointer_TO_r_struct_of_r_int32END | |
| 2023-09-22 17:10:11.654 : INFO: instruction count: 4: __C322collect_distributed_array_table_head_recursive_count.__m342ENCODE_SInt32$_TO_r_int32 | |
| 2023-09-22 17:10:11.655 : INFO: instruction count: 9: __C322collect_distributed_array_table_head_recursive_count.setPartitionIndex | |
| 2023-09-22 17:10:11.655 : INFO: instruction count: 4: __C322collect_distributed_array_table_head_recursive_count.addPartitionRegion | |
| 2023-09-22 17:10:11.655 : INFO: instruction count: 4: __C322collect_distributed_array_table_head_recursive_count.setPool | |
| 2023-09-22 17:10:11.655 : INFO: instruction count: 3: __C322collect_distributed_array_table_head_recursive_count.addHailClassLoader | |
| 2023-09-22 17:10:11.655 : INFO: instruction count: 3: __C322collect_distributed_array_table_head_recursive_count.addFS | |
| 2023-09-22 17:10:11.655 : INFO: instruction count: 4: __C322collect_distributed_array_table_head_recursive_count.addTaskContext | |
| 2023-09-22 17:10:11.686 : INFO: encoder cache hit | |
| 2023-09-22 17:10:11.688 MemoryStore: INFO: Block broadcast_2 stored as values in memory (estimated size 136.0 B, free 24.4 GiB) | |
| 2023-09-22 17:10:11.692 MemoryStore: INFO: Block broadcast_2_piece0 stored as bytes in memory (estimated size 91.0 B, free 24.4 GiB) | |
| 2023-09-22 17:10:11.693 BlockManagerInfo: INFO: Added broadcast_2_piece0 in memory on notebook-slowdown-repro-m.c.broad-ctsa.internal:38207 (size: 91.0 B, free: 24.4 GiB) | |
| 2023-09-22 17:10:11.694 SparkContext: INFO: Created broadcast 2 from broadcast at SparkBackend.scala:361 | |
| 2023-09-22 17:10:11.695 : INFO: instruction count: 3: __C297HailClassLoaderContainer.<init> | |
| 2023-09-22 17:10:11.695 : INFO: instruction count: 3: __C297HailClassLoaderContainer.<clinit> | |
| 2023-09-22 17:10:11.695 : INFO: instruction count: 3: __C299FSContainer.<init> | |
| 2023-09-22 17:10:11.695 : INFO: instruction count: 3: __C299FSContainer.<clinit> | |
| 2023-09-22 17:10:11.709 : INFO: instruction count: 3: __C301Compiled.<init> | |
| 2023-09-22 17:10:11.709 : INFO: instruction count: 31: __C301Compiled.apply | |
| 2023-09-22 17:10:11.710 : INFO: instruction count: 431: __C301Compiled.__m303split_ToArray | |
| 2023-09-22 17:10:11.711 : INFO: instruction count: 371: __C301Compiled.__m305split_TailLoop | |
| 2023-09-22 17:10:11.711 : INFO: instruction count: 4: __C301Compiled.setBackend | |
| 2023-09-22 17:10:11.711 : INFO: instruction count: 9: __C301Compiled.__m351ENCODE_SBaseStructPointer_TO_r_struct_of_r_struct_of_r_int32ANDr_int32ENDEND | |
| 2023-09-22 17:10:11.712 : INFO: instruction count: 25: __C301Compiled.__m352ENCODE_SBaseStructPointer_TO_r_struct_of_r_int32ANDr_int32END | |
| 2023-09-22 17:10:11.712 : INFO: instruction count: 4: __C301Compiled.__m353ENCODE_SInt32$_TO_r_int32 | |
| 2023-09-22 17:10:11.712 : INFO: instruction count: 9: __C301Compiled.__m354ENCODE_SBaseStructPointer_TO_r_struct_of_r_struct_of_ENDEND | |
| 2023-09-22 17:10:11.712 : INFO: instruction count: 1: __C301Compiled.__m355ENCODE_SBaseStructPointer_TO_r_struct_of_END | |
| 2023-09-22 17:10:11.712 : INFO: instruction count: 18: __C301Compiled.__m356str | |
| 2023-09-22 17:10:11.712 : INFO: instruction count: 29: __C301Compiled.__m359concat | |
| 2023-09-22 17:10:11.713 : INFO: instruction count: 29: __C301Compiled.__m360concat | |
| 2023-09-22 17:10:11.713 : INFO: instruction count: 9: __C301Compiled.__m361concat | |
| 2023-09-22 17:10:11.713 : INFO: instruction count: 27: __C301Compiled.__m362DECODE_r_struct_of_r_int32END_TO_SBaseStructPointer | |
| 2023-09-22 17:10:11.713 : INFO: instruction count: 10: __C301Compiled.__m363INPLACE_DECODE_r_int32_TO_r_int32 | |
| 2023-09-22 17:10:11.713 : INFO: instruction count: 70: __C301Compiled.__m367split_StreamFold | |
| 2023-09-22 17:10:11.713 : INFO: instruction count: 11: __C301Compiled.__m377ord_gteq | |
| 2023-09-22 17:10:11.714 : INFO: instruction count: 16: __C301Compiled.__m378ord_gteqNonnull | |
| 2023-09-22 17:10:11.714 : INFO: instruction count: 11: __C301Compiled.__m379ord_lteq | |
| 2023-09-22 17:10:11.714 : INFO: instruction count: 14: __C301Compiled.__m380ord_lteqNonnull | |
| 2023-09-22 17:10:11.714 : INFO: instruction count: 11: __C301Compiled.__m388ord_equiv | |
| 2023-09-22 17:10:11.714 : INFO: instruction count: 14: __C301Compiled.__m389ord_equivNonnull | |
| 2023-09-22 17:10:11.715 : INFO: instruction count: 225: __C301Compiled.__m390split_TailLoop | |
| 2023-09-22 17:10:11.715 : INFO: instruction count: 35: __C301Compiled.__m396arrayref_bounds_check | |
| 2023-09-22 17:10:11.715 : INFO: instruction count: 11: __C301Compiled.__m397ord_lteq | |
| 2023-09-22 17:10:11.715 : INFO: instruction count: 16: __C301Compiled.__m398ord_lteqNonnull | |
| 2023-09-22 17:10:11.715 : INFO: instruction count: 9: __C301Compiled.setPartitionIndex | |
| 2023-09-22 17:10:11.715 : INFO: instruction count: 4: __C301Compiled.addPartitionRegion | |
| 2023-09-22 17:10:11.716 : INFO: instruction count: 4: __C301Compiled.setPool | |
| 2023-09-22 17:10:11.716 : INFO: instruction count: 3: __C301Compiled.addHailClassLoader | |
| 2023-09-22 17:10:11.716 : INFO: instruction count: 3: __C301Compiled.addFS | |
| 2023-09-22 17:10:11.716 : INFO: instruction count: 4: __C301Compiled.addTaskContext | |
| 2023-09-22 17:10:11.716 : INFO: instruction count: 3: __C301Compiled.setObjects | |
| 2023-09-22 17:10:11.716 : INFO: instruction count: 110: __C301Compiled.addAndDecodeLiterals | |
| 2023-09-22 17:10:11.717 : INFO: instruction count: 63: __C301Compiled.__m423DECODE_r_struct_of_r_binaryANDr_binaryANDr_struct_of_ENDANDr_struct_of_r_int32ANDr_int64ENDANDr_array_of_r_struct_of_r_int32ANDr_int32ENDEND_TO_SBaseStructPointer | |
| 2023-09-22 17:10:11.717 : INFO: instruction count: 31: __C301Compiled.__m424INPLACE_DECODE_r_binary_TO_r_string | |
| 2023-09-22 17:10:11.717 : INFO: instruction count: 8: __C301Compiled.__m425INPLACE_DECODE_r_struct_of_END_TO_r_struct_of_END | |
| 2023-09-22 17:10:11.717 : INFO: instruction count: 26: __C301Compiled.__m426INPLACE_DECODE_r_struct_of_r_int32ANDr_int64END_TO_r_tuple_of_r_int32ANDr_int64END | |
| 2023-09-22 17:10:11.717 : INFO: instruction count: 10: __C301Compiled.__m427INPLACE_DECODE_r_int64_TO_r_int64 | |
| 2023-09-22 17:10:11.717 : INFO: instruction count: 58: __C301Compiled.__m428INPLACE_DECODE_r_array_of_r_struct_of_r_int32ANDr_int32END_TO_r_array_of_r_struct_of_r_int32ANDr_int32END | |
| 2023-09-22 17:10:11.717 : INFO: instruction count: 26: __C301Compiled.__m429INPLACE_DECODE_r_struct_of_r_int32ANDr_int32END_TO_r_struct_of_r_int32ANDr_int32END | |
| 2023-09-22 17:10:11.718 : INFO: instruction count: 242: __C301Compiled.__m305split_TailLoop_region4_7 | |
| 2023-09-22 17:10:11.718 : INFO: instruction count: 3: __C430__m305split_TailLoopSpills.<init> | |
| 2023-09-22 17:10:11.719 : INFO: instruction count: 3: __C357staticWrapperClass_1.<init> | |
| 2023-09-22 17:10:11.752 : INFO: [collectDArray|table_head_recursive_count|iteration=0,nParts=4]: executing 2 tasks, contexts size = 24.00 B, globals size = 4.00 B | |
| 2023-09-22 17:10:11.753 MemoryStore: INFO: Block broadcast_3 stored as values in memory (estimated size 64.0 B, free 24.4 GiB) | |
| 2023-09-22 17:10:11.757 MemoryStore: INFO: Block broadcast_3_piece0 stored as bytes in memory (estimated size 49.0 B, free 24.4 GiB) | |
| 2023-09-22 17:10:11.758 BlockManagerInfo: INFO: Added broadcast_3_piece0 in memory on notebook-slowdown-repro-m.c.broad-ctsa.internal:38207 (size: 49.0 B, free: 24.4 GiB) | |
| 2023-09-22 17:10:11.759 SparkContext: INFO: Created broadcast 3 from broadcast at SparkBackend.scala:361 | |
| 2023-09-22 17:10:11.791 MemoryStore: INFO: Block broadcast_4 stored as values in memory (estimated size 302.3 KiB, free 24.4 GiB) | |
| 2023-09-22 17:10:11.814 MemoryStore: INFO: Block broadcast_4_piece0 stored as bytes in memory (estimated size 38.4 KiB, free 24.4 GiB) | |
| 2023-09-22 17:10:11.815 BlockManagerInfo: INFO: Added broadcast_4_piece0 in memory on notebook-slowdown-repro-m.c.broad-ctsa.internal:38207 (size: 38.4 KiB, free: 24.4 GiB) | |
| 2023-09-22 17:10:11.816 SparkContext: INFO: Created broadcast 4 from broadcast at SparkBackend.scala:361 | |
| 2023-09-22 17:10:11.965 SparkContext: INFO: Starting job: collect at SparkBackend.scala:406 | |
| 2023-09-22 17:10:11.982 DAGScheduler: INFO: Got job 0 (collect at SparkBackend.scala:406) with 2 output partitions | |
| 2023-09-22 17:10:11.982 DAGScheduler: INFO: Final stage: ResultStage 0 (collect at SparkBackend.scala:406) | |
| 2023-09-22 17:10:11.982 DAGScheduler: INFO: Parents of final stage: List() | |
| 2023-09-22 17:10:11.984 DAGScheduler: INFO: Missing parents: List() | |
| 2023-09-22 17:10:11.991 DAGScheduler: INFO: Submitting ResultStage 0 ([0] at RDD at SparkBackend.scala:377), which has no missing parents | |
| 2023-09-22 17:10:12.055 MemoryStore: INFO: Block broadcast_5 stored as values in memory (estimated size 12.2 KiB, free 24.4 GiB) | |
| 2023-09-22 17:10:12.060 MemoryStore: INFO: Block broadcast_5_piece0 stored as bytes in memory (estimated size 6.9 KiB, free 24.4 GiB) | |
| 2023-09-22 17:10:12.061 BlockManagerInfo: INFO: Added broadcast_5_piece0 in memory on notebook-slowdown-repro-m.c.broad-ctsa.internal:38207 (size: 6.9 KiB, free: 24.4 GiB) | |
| 2023-09-22 17:10:12.061 SparkContext: INFO: Created broadcast 5 from broadcast at DAGScheduler.scala:1513 | |
| 2023-09-22 17:10:12.079 DAGScheduler: INFO: Submitting 2 missing tasks from ResultStage 0 ([0] at RDD at SparkBackend.scala:377) (first 15 tasks are for partitions Vector(0, 1)) | |
| 2023-09-22 17:10:12.080 YarnScheduler: INFO: Adding task set 0.0 with 2 tasks resource profile 0 | |
| 2023-09-22 17:10:12.103 FairSchedulableBuilder: INFO: Added task set TaskSet_0.0 tasks to pool default | |
| 2023-09-22 17:10:12.120 TaskSetManager: INFO: Starting task 0.0 in stage 0.0 (TID 0) (notebook-slowdown-repro-w-1.c.broad-ctsa.internal, executor 1, partition 0, PROCESS_LOCAL, 15455 bytes) taskResourceAssignments Map() | |
| 2023-09-22 17:10:12.126 TaskSetManager: INFO: Starting task 1.0 in stage 0.0 (TID 1) (notebook-slowdown-repro-w-0.c.broad-ctsa.internal, executor 2, partition 1, PROCESS_LOCAL, 15455 bytes) taskResourceAssignments Map() | |
| 2023-09-22 17:10:12.131 YarnAllocator: INFO: Resource profile 0 doesn't exist, adding it | |
| 2023-09-22 17:10:12.132 YarnAllocator: INFO: Driver requested a total number of 1 executor(s) for resource profile id: 0. | |
| 2023-09-22 17:22:22.737 SparkContext: INFO: Invoking stop() from shutdown hook | |
| 2023-09-22 17:22:22.751 AbstractConnector: INFO: Stopped Spark@4c6e3b0d{HTTP/1.1, (http/1.1)}{0.0.0.0:0} | |
| 2023-09-22 17:22:22.753 SparkUI: INFO: Stopped Spark web UI at http://notebook-slowdown-repro-m.c.broad-ctsa.internal:43055 | |
| 2023-09-22 17:22:22.759 DAGScheduler: INFO: Job 0 failed: collect at SparkBackend.scala:406, took 730.794050 s | |
| 2023-09-22 17:22:22.759 DAGScheduler: INFO: ResultStage 0 (collect at SparkBackend.scala:406) failed in 730.748 s due to Stage cancelled because SparkContext was shut down | |
| 2023-09-22 17:22:22.760 : ERROR: error while applying lowering 'LowerOrInterpretNonCompilable' | |
| 2023-09-22 17:22:22.761 : ERROR: error while applying lowering 'EvalRelationalLets' | |
| 2023-09-22 17:22:22.761 : INFO: RegionPool: initialized for thread 17: Thread-3 | |
| 2023-09-22 17:22:22.761 : INFO: TaskReport: stage=0, partition=0, attempt=0, peakBytes=0, peakBytesReadable=0.00 B, chunks requested=0, cache hits=0 | |
| 2023-09-22 17:22:22.761 : INFO: RegionPool: FREE: 0 allocated (0 blocks / 0 chunks), regions.size = 0, 0 current java objects, thread 17: Thread-3 | |
| 2023-09-22 17:22:22.761 : INFO: RegionPool: FREE: 192.0K allocated (192.0K blocks / 0 chunks), regions.size = 3, 0 current java objects, thread 17: Thread-3 | |
| 2023-09-22 17:22:22.769 YarnClientSchedulerBackend: INFO: Interrupting monitor thread | |
| 2023-09-22 17:22:22.769 YarnClientSchedulerBackend: INFO: Shutting down all executors | |
| 2023-09-22 17:22:22.770 YarnSchedulerBackend$YarnDriverEndpoint: INFO: Asking each executor to shut down | |
| 2023-09-22 17:22:22.777 ApplicationMaster: INFO: Final app status: SUCCEEDED, exitCode: 0 | |
| 2023-09-22 17:22:22.786 ApplicationMaster: INFO: Unregistering ApplicationMaster with SUCCEEDED | |
| 2023-09-22 17:22:22.796 ApplicationMaster: INFO: Deleting staging directory hdfs://notebook-slowdown-repro-m/user/root/.sparkStaging/application_1695402030462_0001 | |
| 2023-09-22 17:22:22.826 YarnClientSchedulerBackend: INFO: YARN client scheduler backend Stopped | |
| 2023-09-22 17:22:23.034 : INFO: RegionPool: initialized for thread 125: Thread-44 | |
| 2023-09-22 17:22:23.046 : INFO: RegionPool: initialized for thread 125: Thread-44 | |
| 2023-09-22 17:22:23.046 : INFO: TaskReport: stage=0, partition=0, attempt=0, peakBytes=0, peakBytesReadable=0.00 B, chunks requested=0, cache hits=0 | |
| 2023-09-22 17:22:23.046 : INFO: RegionPool: FREE: 0 allocated (0 blocks / 0 chunks), regions.size = 0, 0 current java objects, thread 125: Thread-44 | |
| 2023-09-22 17:22:23.046 : INFO: RegionPool: FREE: 64.0K allocated (64.0K blocks / 0 chunks), regions.size = 1, 0 current java objects, thread 125: Thread-44 | |
| 2023-09-22 17:22:23.049 : ERROR: IllegalStateException: Cannot call methods on a stopped SparkContext. | |
| This stopped SparkContext was created at: | |
| org.apache.spark.SparkContext.<init>(SparkContext.scala:87) | |
| is.hail.backend.spark.SparkBackend$.configureAndCreateSparkContext(SparkBackend.scala:145) | |
| is.hail.backend.spark.SparkBackend$.apply(SparkBackend.scala:237) | |
| is.hail.backend.spark.SparkBackend.apply(SparkBackend.scala) | |
| java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) | |
| java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) | |
| java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
| java.base/java.lang.reflect.Method.invoke(Method.java:566) | |
| py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) | |
| py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) | |
| py4j.Gateway.invoke(Gateway.java:282) | |
| py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) | |
| py4j.commands.CallCommand.execute(CallCommand.java:79) | |
| py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) | |
| py4j.ClientServerConnection.run(ClientServerConnection.java:106) | |
| java.base/java.lang.Thread.run(Thread.java:829) | |
| The currently active SparkContext was created at: | |
| org.apache.spark.SparkContext.<init>(SparkContext.scala:87) | |
| is.hail.backend.spark.SparkBackend$.configureAndCreateSparkContext(SparkBackend.scala:145) | |
| is.hail.backend.spark.SparkBackend$.apply(SparkBackend.scala:237) | |
| is.hail.backend.spark.SparkBackend.apply(SparkBackend.scala) | |
| java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) | |
| java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) | |
| java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
| java.base/java.lang.reflect.Method.invoke(Method.java:566) | |
| py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) | |
| py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) | |
| py4j.Gateway.invoke(Gateway.java:282) | |
| py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) | |
| py4j.commands.CallCommand.execute(CallCommand.java:79) | |
| py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) | |
| py4j.ClientServerConnection.run(ClientServerConnection.java:106) | |
| java.base/java.lang.Thread.run(Thread.java:829) | |
| From java.lang.IllegalStateException: Cannot call methods on a stopped SparkContext. | |
| This stopped SparkContext was created at: | |
| org.apache.spark.SparkContext.<init>(SparkContext.scala:87) | |
| is.hail.backend.spark.SparkBackend$.configureAndCreateSparkContext(SparkBackend.scala:145) | |
| is.hail.backend.spark.SparkBackend$.apply(SparkBackend.scala:237) | |
| is.hail.backend.spark.SparkBackend.apply(SparkBackend.scala) | |
| java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) | |
| java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) | |
| java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
| java.base/java.lang.reflect.Method.invoke(Method.java:566) | |
| py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) | |
| py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) | |
| py4j.Gateway.invoke(Gateway.java:282) | |
| py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) | |
| py4j.commands.CallCommand.execute(CallCommand.java:79) | |
| py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) | |
| py4j.ClientServerConnection.run(ClientServerConnection.java:106) | |
| java.base/java.lang.Thread.run(Thread.java:829) | |
| The currently active SparkContext was created at: | |
| org.apache.spark.SparkContext.<init>(SparkContext.scala:87) | |
| is.hail.backend.spark.SparkBackend$.configureAndCreateSparkContext(SparkBackend.scala:145) | |
| is.hail.backend.spark.SparkBackend$.apply(SparkBackend.scala:237) | |
| is.hail.backend.spark.SparkBackend.apply(SparkBackend.scala) | |
| java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) | |
| java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) | |
| java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
| java.base/java.lang.reflect.Method.invoke(Method.java:566) | |
| py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) | |
| py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) | |
| py4j.Gateway.invoke(Gateway.java:282) | |
| py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) | |
| py4j.commands.CallCommand.execute(CallCommand.java:79) | |
| py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) | |
| py4j.ClientServerConnection.run(ClientServerConnection.java:106) | |
| java.base/java.lang.Thread.run(Thread.java:829) | |
| at org.apache.spark.SparkContext.assertNotStopped(SparkContext.scala:120) | |
| at org.apache.spark.SparkContext.defaultParallelism(SparkContext.scala:2532) | |
| at is.hail.backend.spark.SparkBackend.defaultParallelism(SparkBackend.scala:412) | |
| at is.hail.expr.ir.IRParser$.$anonfun$table_ir_1$27(Parser.scala:1714) | |
| at scala.runtime.java8.JFunction0$mcI$sp.apply(JFunction0$mcI$sp.java:23) | |
| at scala.Option.getOrElse(Option.scala:189) | |
| at is.hail.expr.ir.IRParser$.table_ir_1(Parser.scala:1714) | |
| at is.hail.expr.ir.IRParser$.$anonfun$table_ir$1(Parser.scala:1602) | |
| at is.hail.utils.StackSafe$More.advance(StackSafe.scala:64) | |
| at is.hail.utils.StackSafe$.run(StackSafe.scala:16) | |
| at is.hail.utils.StackSafe$StackFrame.run(StackSafe.scala:32) | |
| at is.hail.expr.ir.IRParser$.$anonfun$parse_value_ir$1(Parser.scala:2157) | |
| at is.hail.expr.ir.IRParser$.parse(Parser.scala:2153) | |
| at is.hail.expr.ir.IRParser$.parse_value_ir(Parser.scala:2157) | |
| at is.hail.backend.spark.SparkBackend.$anonfun$parse_value_ir$2(SparkBackend.scala:691) | |
| at is.hail.backend.ExecuteContext$.$anonfun$scoped$3(ExecuteContext.scala:76) | |
| at is.hail.utils.package$.using(package.scala:637) | |
| at is.hail.backend.ExecuteContext$.$anonfun$scoped$2(ExecuteContext.scala:76) | |
| at is.hail.utils.package$.using(package.scala:637) | |
| at is.hail.annotations.RegionPool$.scoped(RegionPool.scala:17) | |
| at is.hail.backend.ExecuteContext$.scoped(ExecuteContext.scala:62) | |
| at is.hail.backend.spark.SparkBackend.$anonfun$withExecuteContext$1(SparkBackend.scala:345) | |
| at is.hail.backend.spark.SparkBackend.$anonfun$parse_value_ir$1(SparkBackend.scala:690) | |
| at is.hail.utils.ExecutionTimer$.time(ExecutionTimer.scala:52) | |
| at is.hail.utils.ExecutionTimer$.logTime(ExecutionTimer.scala:59) | |
| at is.hail.backend.spark.SparkBackend.parse_value_ir(SparkBackend.scala:689) | |
| at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) | |
| at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) | |
| at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) | |
| at java.base/java.lang.reflect.Method.invoke(Method.java:566) | |
| at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) | |
| at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) | |
| at py4j.Gateway.invoke(Gateway.java:282) | |
| at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) | |
| at py4j.commands.CallCommand.execute(CallCommand.java:79) | |
| at py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) | |
| at py4j.ClientServerConnection.run(ClientServerConnection.java:106) | |
| at java.base/java.lang.Thread.run(Thread.java:829) | |
| 2023-09-22 17:22:23.222 MapOutputTrackerMasterEndpoint: INFO: MapOutputTrackerMasterEndpoint stopped! | |
| 2023-09-22 17:22:23.243 MemoryStore: INFO: MemoryStore cleared | |
| 2023-09-22 17:22:23.244 BlockManager: INFO: BlockManager stopped | |
| 2023-09-22 17:22:23.251 BlockManagerMaster: INFO: BlockManagerMaster stopped | |
| 2023-09-22 17:22:23.257 StatsdSink: INFO: StatsdSink stopped. | |
| 2023-09-22 17:22:23.263 OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: INFO: OutputCommitCoordinator stopped! | |
| 2023-09-22 17:22:23.273 SparkContext: INFO: Successfully stopped SparkContext |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment