Patch next with PR adds experimental serverOnlyDependencies property https://github.com/vercel/next.js/pull/65415

next.config.mjs

import { withPayload } from '@payloadcms/next/withPayload';
import { resolve } from 'path';

/** @type {import('next').NextConfig} */
const nextConfig = {
  experimental: {
    serverOnlyDependencies: [resolve(import.meta.dirname, 'payload.config.proxy.ts')],
  },
};

export default withPayload(nextConfig);

next@14.3.0-canary.68.patch

diff --git a/dist/build/webpack/plugins/flight-client-entry-plugin.d.ts b/dist/build/webpack/plugins/flight-client-entry-plugin.d.ts
index 8f857af48d72f7b9f47f06c00748af5a14723271..860094a88efe78577860a51c8c27292abd7535ff 100644
--- a/dist/build/webpack/plugins/flight-client-entry-plugin.d.ts
+++ b/dist/build/webpack/plugins/flight-client-entry-plugin.d.ts
@@ -6,6 +6,7 @@ interface Options {
     appDir: string;
     isEdgeServer: boolean;
     encryptionKey: string;
+    ignore: string[]
 }
 type Actions = {
     [actionId: string]: {
@@ -25,6 +26,7 @@ export type ActionManifest = {
 export declare class FlightClientEntryPlugin {
     dev: boolean;
     appDir: string;
+    ignore: string[]
     encryptionKey: string;
     isEdgeServer: boolean;
     assetPrefix: string;
diff --git a/dist/build/webpack/plugins/flight-client-entry-plugin.js b/dist/build/webpack/plugins/flight-client-entry-plugin.js
index 6d3991896e21d38068c6c21a0592277de5d8471b..89e810bc5261dd5776745a70db6c5b933c3819b5 100644
--- a/dist/build/webpack/plugins/flight-client-entry-plugin.js
+++ b/dist/build/webpack/plugins/flight-client-entry-plugin.js
@@ -106,6 +106,7 @@ class FlightClientEntryPlugin {
     constructor(options){
         this.dev = options.dev;
         this.appDir = options.appDir;
+        this.ignore = options.ignore
         this.isEdgeServer = options.isEdgeServer;
         this.assetPrefix = !this.dev && !this.isEdgeServer ? '../' : '';
         this.encryptionKey = options.encryptionKey;
@@ -428,7 +429,7 @@ class FlightClientEntryPlugin {
             if ((_mod_matchResource = mod.matchResource) == null ? void 0 : _mod_matchResource.startsWith(_constants1.BARREL_OPTIMIZATION_PREFIX)) {
                 modRequest = mod.matchResource + ':' + modRequest;
             }
-            if (!modRequest) return;
+            if (!modRequest || this.ignore.includes(modRequest)) return;
             if (visited.has(modRequest)) {
                 if (clientComponentImports[modRequest]) {
                     addClientImport(mod, modRequest, clientComponentImports, importedIdentifiers, false);
diff --git a/dist/build/webpack-config.js b/dist/build/webpack-config.js
index 8cee018bbdbb82c0f84ca1659307155f2f73f360..e7c759a9c1071a1697b82de24a924b7460b31198 100644
--- a/dist/build/webpack-config.js
+++ b/dist/build/webpack-config.js
@@ -1518,7 +1518,8 @@ async function getBaseWebpackConfig(dir, { buildId, encryptionKey, config, compi
                 appDir,
                 dev,
                 isEdgeServer,
-                encryptionKey
+                encryptionKey,
+                ignore: config.experimental.serverOnlyDependencies ?? [],
             })),
             hasAppDir && !isClient && new _nexttypesplugin.NextTypesPlugin({
                 dir,
diff --git a/dist/server/config-schema.js b/dist/server/config-schema.js
index 86fbccfca8f53d674b4fc880b9d70e2b2d5233f6..c5808c4c864e055cacabee4aea537e372c7522c2 100644
--- a/dist/server/config-schema.js
+++ b/dist/server/config-schema.js
@@ -281,6 +281,7 @@ const configSchema = _zod.z.lazy(()=>_zod.z.strictObject({
             taint: _zod.z.boolean().optional(),
             prerenderEarlyExit: _zod.z.boolean().optional(),
             proxyTimeout: _zod.z.number().gte(0).optional(),
+            serverOnlyDependencies: _zod.z.array(_zod.z.string()).optional(),
             scrollRestoration: _zod.z.boolean().optional(),
             sri: _zod.z.object({
                 algorithm: _zod.z.enum([
diff --git a/dist/server/config-shared.d.ts b/dist/server/config-shared.d.ts
index c196d0f28ab2c83a6e1b1357241a2734ca93ee5f..1ae2ade2a63a4a9dfacee0440d5eef1581f522ee 100644
--- a/dist/server/config-shared.d.ts
+++ b/dist/server/config-shared.d.ts
@@ -235,6 +235,10 @@ export interface ExperimentalConfig {
     adjustFontFallbacks?: boolean;
     adjustFontFallbacksWithSizeAdjust?: boolean;
     webVitalsAttribution?: Array<(typeof WEB_VITALS)[number]>;
+    /**
+     * A list of imports that should completely disable any client dependencies from being added to generated client JS.
+    */
+    serverOnlyDependencies?: string[]
     /**
      * Automatically apply the "modularizeImports" optimization to imports of the specified packages.
      */

package.json

 "pnpm": {
    "patchedDependencies": {
      "next@14.3.0-canary.68": "patches/next@14.3.0-canary.68.patch"
    }
  }

payload.config.proxy.ts

// import config from here instead of @payload-config outside of app/(admin)
import config from './payload.config';

export default config;

Thanks @r1tsuu!

For anyone else who wants to do this against stable Next, this is my patch applied to next@14.2.3:

diff --git a/dist/build/webpack/plugins/flight-client-entry-plugin.d.ts b/dist/build/webpack/plugins/flight-client-entry-plugin.d.ts
index f2619cff892a2714b6f624de6e6633c53ca00632..86035c93bd980bb330c6fb10e27d42527ad45691 100644
--- a/dist/build/webpack/plugins/flight-client-entry-plugin.d.ts
+++ b/dist/build/webpack/plugins/flight-client-entry-plugin.d.ts
@@ -5,6 +5,7 @@ interface Options {
     appDir: string;
     isEdgeServer: boolean;
     encryptionKey: string;
+    ignore: string[];
 }
 type Actions = {
     [actionId: string]: {
@@ -24,6 +25,7 @@ export type ActionManifest = {
 export declare class FlightClientEntryPlugin {
     dev: boolean;
     appDir: string;
+    ignore: string[];
     encryptionKey: string;
     isEdgeServer: boolean;
     assetPrefix: string;
diff --git a/dist/build/webpack/plugins/flight-client-entry-plugin.js b/dist/build/webpack/plugins/flight-client-entry-plugin.js
index adc89c5ffc47a71ec641a9ccc8eddf920e0ae711..f4f0b17d714e5acb058dba92a42f05778f75bc9e 100644
--- a/dist/build/webpack/plugins/flight-client-entry-plugin.js
+++ b/dist/build/webpack/plugins/flight-client-entry-plugin.js
@@ -106,6 +106,7 @@ class FlightClientEntryPlugin {
     constructor(options){
         this.dev = options.dev;
         this.appDir = options.appDir;
+        this.ignore = options.ignore;
         this.isEdgeServer = options.isEdgeServer;
         this.assetPrefix = !this.dev && !this.isEdgeServer ? "../" : "";
         this.encryptionKey = options.encryptionKey;
@@ -431,7 +432,7 @@ class FlightClientEntryPlugin {
             if ((_mod_matchResource = mod.matchResource) == null ? void 0 : _mod_matchResource.startsWith(_constants1.BARREL_OPTIMIZATION_PREFIX)) {
                 modRequest = mod.matchResource + ":" + modRequest;
             }
-            if (!modRequest) return;
+            if (!modRequest || this.ignore.includes(modRequest)) return;
             if (visited.has(modRequest)) {
                 if (clientComponentImports[modRequest]) {
                     addClientImport(mod, modRequest, clientComponentImports, importedIdentifiers, false);
diff --git a/dist/build/webpack-config.js b/dist/build/webpack-config.js
index 5a1203af9dcc1209d21a3da5f5b5d5b5ef0256aa..1bd0347cdeb0b3df8931ddc49bfd0178470939f6 100644
--- a/dist/build/webpack-config.js
+++ b/dist/build/webpack-config.js
@@ -1520,7 +1520,8 @@ async function getBaseWebpackConfig(dir, { buildId, encryptionKey, config, compi
                 appDir,
                 dev,
                 isEdgeServer,
-                encryptionKey
+                encryptionKey,
+                ignore: config.experimental.serverOnlyDependencies ?? [],
             })),
             hasAppDir && !isClient && new _nexttypesplugin.NextTypesPlugin({
                 dir,
diff --git a/dist/server/config-schema.js b/dist/server/config-schema.js
index da7845894c912114ee107a2e37b4857eb55f43bd..ca0300715022e369b4f93ab87b8f379b60c8c61a 100644
--- a/dist/server/config-schema.js
+++ b/dist/server/config-schema.js
@@ -276,6 +276,7 @@ const configSchema = _zod.z.lazy(()=>_zod.z.strictObject({
             taint: _zod.z.boolean().optional(),
             prerenderEarlyExit: _zod.z.boolean().optional(),
             proxyTimeout: _zod.z.number().gte(0).optional(),
+            serverOnlyDependencies: _zod.z.array(_zod.z.string()).optional(),
             serverComponentsExternalPackages: _zod.z.array(_zod.z.string()).optional(),
             scrollRestoration: _zod.z.boolean().optional(),
             sri: _zod.z.object({
diff --git a/dist/server/config-shared.d.ts b/dist/server/config-shared.d.ts
index 11114bda9f89f599041834174ca885ffe78613cb..203958052e334f6cd3a8d1fa3cdec30ec83f2f05 100644
--- a/dist/server/config-shared.d.ts
+++ b/dist/server/config-shared.d.ts
@@ -234,6 +234,10 @@ export interface ExperimentalConfig {
      */
     serverComponentsExternalPackages?: string[];
     webVitalsAttribution?: Array<(typeof WEB_VITALS)[number]>;
+    /**
+     * A list of imports that should completely disable any client dependencies from being added to generated client JS.
+     */
+    serverOnlyDependencies?: string[];
     /**
      * Automatically apply the "modularizeImports" optimization to imports of the specified packages.
      */

thank you @r1tsuu @richardbutler!