GestureDetection onDrag OK

main.dart

import 'package:flutter/gestures.dart';
import 'package:flutter/services.dart';
import 'package:flutter/material.dart';

void main() {
  runApp(PanIssueDemo());
}

class PanIssueDemo extends StatelessWidget {
  @override
  Widget build(BuildContext context) {
    // On a device this [setSystemUIOverlayStyle] call will make your AppBar cool
    // on Android. It also helps with the AppBar effect shown here just for fun.
    // By also making the transparent gradient AppBar visible on the top system
    // status icons and it also makes it so that the AppBar and status icons area
    // always uses the same color as the one used in Flutter's AppBar
    // and not standard Android two toned one, so it looks more like an iPhone
    SystemChrome.setSystemUIOverlayStyle(
      SystemUiOverlayStyle(
        systemNavigationBarColor: Colors.grey[100],
        statusBarColor: Colors.transparent,
        statusBarIconBrightness: Brightness.light,
        systemNavigationBarIconBrightness: Brightness.dark,
      ),
    );

    return MaterialApp(
      theme: ThemeData(
        primarySwatch: Colors.indigo,
        scaffoldBackgroundColor: Colors.grey[100],
        buttonTheme: ButtonThemeData(
          colorScheme: ColorScheme.fromSwatch(primarySwatch: Colors.indigo),
          textTheme: ButtonTextTheme.primary,
        ),
      ),
      debugShowCheckedModeBanner: false,
      home: HomePage(),
    );
  }
}

class HomePage extends StatefulWidget {
  @override
  State<StatefulWidget> createState() {
    return _HomePageState();
  }
}

class _HomePageState extends State<HomePage> {
  final Color _boxColor = Colors.blue;
  bool _showMoreWidgets = false;

  @override
  Widget build(BuildContext context) {
    return Scaffold(
      extendBodyBehindAppBar: true,
      extendBody: true,
      appBar: AppBar(
        title: const Text('Drag on a Scrolling Surface'),
        centerTitle: true,
        elevation: 0,
        backgroundColor: Colors.transparent,

        // Fancy gradient partially transparent AppBar
        flexibleSpace: Container(
          decoration: BoxDecoration(
            gradient: LinearGradient(
              begin: Alignment.topLeft,
              end: Alignment.topRight,
              colors: [
                Colors.indigo,
                Colors.indigo.withOpacity(0.7),
              ],
            ),
          ),
          child: null,
        ),
      ),
      body: Scrollbar(
        child: CustomScrollView(
          slivers: <Widget>[
            SliverPadding(
              padding: const EdgeInsets.fromLTRB(15, 0, 15, 0),
              sliver: SliverList(
                delegate: SliverChildListDelegate(
                  [
                    // We need to add back the padding we removed by
                    // allowing scrolling under the toolbar
                    SizedBox(
                        height: MediaQuery.of(context).padding.top +
                            kToolbarHeight),
                    // And then some gap space too
                    const SizedBox(height: 20),

                    Center(
                        child: Text('Drag in the blue box\n(OK onDrag version)',
                            style: Theme.of(context).textTheme.headline5)),
                    const Center(
                        child: SizedBox(
                      width: 450,
                      child: Text(
                          'When you manage to start the dragging the postion text info in '
                          'the box will update as you move the cursor or touch point around.'),
                    )),
                    const SizedBox(height: 10),

                    Center(
                        child: SizedBox(
                            width: 250,
                            height: 130,
                            child: GestureBoxDemo(color: _boxColor))),

                    const SizedBox(height: 10),
                    Center(
                      child: SizedBox(
                        width: 450,
                        child: SwitchListTile(
                          title: const Text('Show grid items'),
                          subtitle: const Text(
                              "When you turn on grid items, the surface will scroll, but "
                              "when using onHorizontalDrag and onVerticalDrag event handlers "
                              "instead of onPan, the Widget's gesture detector will get the events "
                              "instead of the scroll handler. All works OK now."),
                          value: _showMoreWidgets,
                          onChanged: (value) {
                            setState(() {
                              _showMoreWidgets = value;
                            });
                          },
                        ),
                      ),
                    ),

                    const SizedBox(height: 10),
                  ],
                ),
              ),
            ),

            // If we show more Widgets,
            // then we build some grid items as another SliverList item
            // just to make sure we have long scrollable sliver list
            if (_showMoreWidgets) const MoreWidgets(),
          ],
        ),
      ),
    );
  }
}

// *****************************************************************************

class GestureBoxDemo extends StatefulWidget {
  const GestureBoxDemo({Key key, this.color}) : super(key: key);
  final Color color;

  @override
  _GestureBoxDemoState createState() => _GestureBoxDemoState();
}

class _GestureBoxDemoState extends State<GestureBoxDemo> {
  final GlobalKey _boxKey = GlobalKey();

  String _panInfoFromSide = '';
  String _panInfoStart = '';
  String _panInfoPos = '';
  String _panInfoDiff = '';

  @override
  void initState() {
    super.initState();
  }

  @override
  void didUpdateWidget(GestureBoxDemo oldWidget) {
    super.didUpdateWidget(oldWidget);
  }

  Offset getOffset(Offset ratio) {
    final RenderBox renderBox =
        _boxKey.currentContext.findRenderObject() as RenderBox;
    final Offset startPosition = renderBox.localToGlobal(Offset.zero);
    return ratio - startPosition;
  }

  void onStart(Offset offset) {
    final RenderBox _renderBox =
        _boxKey.currentContext.findRenderObject() as RenderBox;
    final Size _size = _renderBox.size;
    final Offset _boxStart = _renderBox.localToGlobal(Offset.zero);
    final double _diff = offset.dx - _boxStart.dx - _size.width / 2;
    setState(() {
      _panInfoStart = 'Start pos: $offset';
      _panInfoPos = 'Position: $offset';
      _panInfoDiff = 'Box width center diff: ${_diff.toStringAsFixed(1)}';
      if (_diff < 0) {
        _panInfoFromSide = 'Start from box left side';
      } else {
        _panInfoFromSide = 'Start from box right side';
      }
    });
  }

  void onUpdate(Offset offset) {
    final RenderBox renderBox =
        _boxKey.currentContext.findRenderObject() as RenderBox;
    final Size size = renderBox.size;
    final Offset _position = renderBox.localToGlobal(Offset.zero);
    final double _diff = offset.dx - _position.dx - size.width / 2;
    setState(() {
      _panInfoPos = 'Position: $offset';
      _panInfoDiff = 'Box width center diff: ${_diff.toStringAsFixed(1)}';
    });
  }

  void onEnd() {
    setState(() {
      _panInfoStart = '';
      _panInfoPos = '';
      _panInfoDiff = '';
      _panInfoFromSide = '';
    });
  }

  @override
  Widget build(BuildContext context) {
    final Color _textColor =
        ThemeData.estimateBrightnessForColor(widget.color) == Brightness.light
            ? Colors.black87
            : Colors.white70;

    return GestureDetector(
      onVerticalDragStart: (details) => onStart(details.globalPosition),
      onVerticalDragUpdate: (details) => onUpdate(details.globalPosition),
      onVerticalDragEnd: (_) => onEnd(),
      onHorizontalDragStart: (details) => onStart(details.globalPosition),
      onHorizontalDragUpdate: (details) => onUpdate(details.globalPosition),
      onHorizontalDragEnd: (_) => onEnd(),
      behavior: HitTestBehavior.opaque,
      dragStartBehavior: DragStartBehavior.down,
      child: Container(
        key: _boxKey,
        color: widget.color,
        child: Column(
          mainAxisAlignment: MainAxisAlignment.center,
          children: [
            Text(_panInfoFromSide, style: TextStyle(color: _textColor)),
            Text(_panInfoStart, style: TextStyle(color: _textColor)),
            Text(_panInfoPos, style: TextStyle(color: _textColor)),
            Text(_panInfoDiff, style: TextStyle(color: _textColor)),
          ],
        ),
      ),
    );
  }
}

// *****************************************************************************

class MoreWidgets extends StatelessWidget {
  const MoreWidgets({Key key}) : super(key: key);

  @override
  Widget build(BuildContext context) {
    final _gridItems = List<GridItem>.generate(
      200,
      (index) {
        return GridItem(
            title: 'Tile nr ${index + 1}',
            color: Colors.primaries[index % Colors.primaries.length][800]);
      },
    );

    return SliverPadding(
      padding: const EdgeInsets.fromLTRB(15, 0, 15, 0),
      sliver: SliverGrid(
        gridDelegate: const SliverGridDelegateWithFixedCrossAxisCount(
          crossAxisCount: 2,
          mainAxisSpacing: 15,
          crossAxisSpacing: 15,
          childAspectRatio: 2,
        ),
        delegate: SliverChildBuilderDelegate(
          (ctx, index) {
            return Card(
              elevation: 6,
              child: _gridItems[index],
            );
          },
          childCount: _gridItems.length,
        ),
      ),
    );
  }
}

// *****************************************************************************

class GridItem extends StatelessWidget {
  const GridItem({Key key, this.title, this.color, this.height, this.bodyText})
      : super(key: key);

  final String title;
  final Color color;
  final double height;
  final String bodyText;

  @override
  Widget build(BuildContext context) {
    return Container(
      color: color,
      padding: const EdgeInsets.all(10),
      child: Column(
        children: <Widget>[
          Text(
            title,
            style: const TextStyle(
              color: Colors.white,
              fontSize: 18,
            ),
          ),
          if (height != null && height > 0) SizedBox(height: height),
          if (height != null && height > 0)
            Text(bodyText,
                style: const TextStyle(
                    color: Colors.white, fontWeight: FontWeight.bold)),
        ],
      ),
    );
  }
}

This example shows an alternative way by using onDrag events instead of the onPan event handlers to make a widget with gesture detector that works when the Widget is on a scrolling surface.

This gist is one possible workaround for the issue presented here:
flutter/flutter#50776

See and try the issue in
DartPad: https://dartpad.dartlang.org/adb2de511815c300b66ad0f4ca76bb63
CodePen: https://codepen.io/rydmike/pen/RwWqQGL

The none functional version that this example works is presented in this gist:
https://gist.github.com/rydmike/adc00ee96cf8cf454790e10c068e2ccc