PhysiGo/app/lib/exercises/exercises_validation/widgets/pose_detector.dart

import 'dart:async';
import 'dart:io';

import 'package:body_detection/models/point3d.dart';
import 'package:body_detection/models/pose_landmark.dart';
import 'package:body_detection/models/pose_landmark_type.dart';
import 'package:rxdart/rxdart.dart';
import 'package:body_detection/body_detection.dart';
import 'package:body_detection/models/image_result.dart';
import 'package:body_detection/models/pose.dart';
import 'package:flutter/material.dart';
import 'package:path_provider/path_provider.dart';

import 'pose_painter.dart';

typedef MeanFilteredData = Iterable<List<double>>;
typedef LandmarkVariations = List<List<double>>;

class PoseDetector extends StatefulWidget {
  const PoseDetector({Key? key}) : super(key: key);

  @override
  State<PoseDetector> createState() => _PoseDetectorState();
}

class _PoseDetectorState extends State<PoseDetector> {
  static const buffer = 10;
  static const _shouldWriteToFile = false;
  final StreamController<Pose> _streamController = StreamController.broadcast();
  final Directory appDir = Directory('/storage/emulated/0/Android/data/com.example.physigo/files');
  Image? _cameraImage;
  Pose? _detectedPose;
  Size _imageSize = Size.zero;
  late Future<void> _startCamera;
  late Stream<LandmarkVariations> _variationsStream;
  late Stream<MeanFilteredData> _meanFilterStream;
  StreamController<Color> _stepExerciseStream = StreamController.broadcast();

  @override
  initState() {
    super.initState();
    _startCamera = _startCameraStream();
    _meanFilterStream = _getMeanFilterStream(_streamController.stream);
    if (_shouldWriteToFile) {
      _writeDataToFile(_meanFilterStream);
    }
    _variationsStream = _meanFilterStream.pairwise().map(_calculateVariations);
  }

  LandmarkVariations _calculateVariations(Iterable<MeanFilteredData> pairPositions) {
    final previous = pairPositions.first.toList();
    final current = pairPositions.last.toList();
    LandmarkVariations variations = [];
    for (int landmark = 0; landmark < previous.length; landmark++) {
      final dx = current[landmark][0] - previous[landmark][0];
      final dy = current[landmark][1] - previous[landmark][1];
      final dz = current[landmark][2] - previous[landmark][2];
      variations.add([dx.roundToDouble(), dy.roundToDouble(), dz.roundToDouble()]);
    }
    return variations;
  }

  void _writeDataToFile(Stream<MeanFilteredData> stream) {
    File meanFilteredData = File("${appDir.path}/meanFilteredData.csv");
    if (meanFilteredData.existsSync()) meanFilteredData.deleteSync();
    stream.listen((meanPositions) {
      for (var position in meanPositions) {
        final str = "${position[0]}, ${position[1]}, ${position[2]};";
        meanFilteredData.writeAsStringSync(str, mode: FileMode.append);
      }
      meanFilteredData.writeAsStringSync("\n", mode: FileMode.append);
    });
  }

  Stream<MeanFilteredData> _getMeanFilterStream(Stream<Pose> stream) {
    return stream
        .where((pose) => pose.landmarks.isNotEmpty)
        .map((pose) => pose.landmarks.where((landmark) => authorizedType.contains(landmark.type)).toList())
        // Get last [buffer] poses
        .bufferCount(buffer, 1)
        // Swap matrix [buffer] * [authorizedType.length]
        .map(_swapMatrixDimensions)
        // For every landmarks, get meanFilter of size [buffer]
        .map((filteredLandmarks) => filteredLandmarks.map(_meanFilter));
  }

  List<double> _meanFilter(List<PoseLandmark> landmarks) {
    return landmarks
        .map((landmark) => landmark.position)
        .map((position) => [
              position.x / buffer,
              position.y / buffer,
              position.z / buffer,
            ])
        .reduce((value, element) => [
              value[0] + element[0],
              value[1] + element[1],
              value[2] + element[2],
            ]);
  }

  List<List<T>> _swapMatrixDimensions<T>(List<List<T>> matrix) {
    final height = matrix.length;
    final width = matrix[0].length;
    List<List<T>> newMatrix = [];
    for (int col = 0; col < width; col++) {
      List<T> newRow = [];
      for (int row = 0; row < height; row++) {
        newRow.add(matrix[row][col]);
      }
      newMatrix.add(newRow);
    }
    return newMatrix;
  }

  Future<void> _startCameraStream() async {
    await BodyDetection.startCameraStream(onFrameAvailable: _handleCameraImage, onPoseAvailable: _handlePose);
    await BodyDetection.enablePoseDetection();
  }

  Future<void> _stopCameraStream() async {
    await BodyDetection.disablePoseDetection();
    await BodyDetection.stopCameraStream();
  }

  void _handleCameraImage(ImageResult result) {
    if (!mounted) return;

    // To avoid a memory leak issue.
    // https://github.com/flutter/flutter/issues/60160
    PaintingBinding.instance?.imageCache?.clear();
    PaintingBinding.instance?.imageCache?.clearLiveImages();

    final image = Image.memory(
      result.bytes,
      gaplessPlayback: true,
      fit: BoxFit.contain,
    );

    setState(() {
      _cameraImage = image;
      _imageSize = result.size;
    });
  }

  void _handlePose(Pose? pose) {
    if (!mounted) return;
    if (pose != null) _streamController.add(pose);
    setState(() {
      _detectedPose = pose;
    });
  }

  @override
  void dispose() {
    _stopCameraStream();
    _streamController.close();
    super.dispose();
  }

  @override
  Widget build(BuildContext context) {
    return FutureBuilder<void>(
      future: _startCamera,
      builder: (context, snapshot) {
        if (snapshot.connectionState == ConnectionState.waiting) {
          return const Center(child: CircularProgressIndicator());
        }
        return Column(
          children: [
            Center(
              child: CustomPaint(
                // size: _imageSize,
                child: _cameraImage,
                foregroundPainter: PosePainter(
                  pose: _detectedPose,
                  imageSize: _imageSize,
                ),
              ),
            ),
            StreamBuilder<MeanFilteredData>(
              stream: _meanFilterStream,
              builder: (context, snapshot) {
                if (!snapshot.hasData) {
                  return CircularProgressIndicator();
                }
                final landmarks = snapshot.data!.toList();
                final xRightHip = landmarks[8][0];
                final xRightKnee = landmarks[10][0];
                final xDistanceHipKnee = (xRightHip - xRightKnee).abs();

                final yRightHip = landmarks[8][1];
                final yRightKnee = landmarks[10][1];
                final yDistanceHipKnee = (yRightHip - yRightKnee).abs();
                var message = "IN BETWEEN";
                if (xDistanceHipKnee < 30) {
                  message = "START";
                  _stepExerciseStream.add(Colors.green);
                } else if (yDistanceHipKnee < 40) {
                  message = "END";
                  _stepExerciseStream.add(Colors.red);
                } else {
                  _stepExerciseStream.add(Colors.yellow);
                }

                final zRightHip = landmarks[8][2];
                final zRightKnee = landmarks[10][2];
                final zDistanceHipKnee = (zRightHip - zRightKnee).abs();

                return Text("$zDistanceHipKnee", style: TextStyle(fontSize: 40));
              },
            ),
            StreamBuilder<Color>(
              stream: _stepExerciseStream.stream,
              builder: (context, snapshot) {
                if (!snapshot.hasData) {
                  return CircularProgressIndicator();
                }
                return Container(
                  height: 100,
                  width: 100,
                  color: snapshot.data!,
                );
              },
            ),
          ],
        );
      },
    );
  }

  static const authorizedType = [
    PoseLandmarkType.nose,
    PoseLandmarkType.leftShoulder,
    PoseLandmarkType.rightShoulder,
    PoseLandmarkType.leftElbow,
    PoseLandmarkType.rightElbow,
    PoseLandmarkType.leftWrist,
    PoseLandmarkType.rightWrist,
    PoseLandmarkType.leftHip,
    PoseLandmarkType.rightHip,
    PoseLandmarkType.leftKnee,
    PoseLandmarkType.rightKnee,
    PoseLandmarkType.leftAnkle,
    PoseLandmarkType.rightAnkle,
  ];
}


/*

GETTING IN POSITION:
  CHECK IF EVERY NECESSARY JOINT ARE ON SCREEN (reliability > 0.8)
  CHECK IF START POSITION IS OKAY (for squat, if knee and hip on same x coordinate)

COUTING REPETITION:
  FROM BEGINNING TO END:
    - BEGINNING: defined by start position, get position of interesting joint
    - END: defined by positions/distance interesting joints (knee and hip same level for squat,
          elbow and should same level for push up)
 */