Xenova/yolov8x-pose

YOLOv8x-pose with ONNX weights to be compatible with Transformers.js.

Usage (Transformers.js)

If you haven't already, you can install the Transformers.js JavaScript library from NPM using:

npm i @xenova/transformers

Example: Perform pose-estimation w/ Xenova/yolov8x-pose.

import { AutoModel, AutoProcessor, RawImage } from '@xenova/transformers';

// Load model and processor
const model_id = 'Xenova/yolov8x-pose';
const model = await AutoModel.from_pretrained(model_id);
const processor = await AutoProcessor.from_pretrained(model_id);

// Read image and run processor
const url = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/football-match.jpg';
const image = await RawImage.read(url);
const { pixel_values } = await processor(image);

// Set thresholds
const threshold = 0.3; // Remove detections with low confidence
const iouThreshold = 0.5; // Used to remove duplicates
const pointThreshold = 0.3; // Hide uncertain points

// Predict bounding boxes and keypoints
const { output0 } = await model({ images: pixel_values });

// Post-process:
const permuted = output0[0].transpose(1, 0);
// `permuted` is a Tensor of shape [ 8400, 56 ]:
// - 8400 potential detections
// - 56 parameters for each box:
//   - 4 for the bounding box dimensions (x-center, y-center, width, height)
//   - 1 for the confidence score
//   - 17 * 3 = 51 for the pose keypoints: 17 labels, each with (x, y, visibilitiy)

// Example code to format it nicely:
const results = [];
const [scaledHeight, scaledWidth] = pixel_values.dims.slice(-2);
for (const [xc, yc, w, h, score, ...keypoints] of permuted.tolist()) {
    if (score < threshold) continue;

    // Get pixel values, taking into account the original image size
    const x1 = (xc - w / 2) / scaledWidth * image.width;
    const y1 = (yc - h / 2) / scaledHeight * image.height;
    const x2 = (xc + w / 2) / scaledWidth * image.width;
    const y2 = (yc + h / 2) / scaledHeight * image.height;
    results.push({ x1, x2, y1, y2, score, keypoints })
}


// Define helper functions
function removeDuplicates(detections, iouThreshold) {
    const filteredDetections = [];

    for (const detection of detections) {
        let isDuplicate = false;
        let duplicateIndex = -1;
        let maxIoU = 0;

        for (let i = 0; i < filteredDetections.length; ++i) {
            const filteredDetection = filteredDetections[i];
            const iou = calculateIoU(detection, filteredDetection);
            if (iou > iouThreshold) {
                isDuplicate = true;
                if (iou > maxIoU) {
                    maxIoU = iou;
                    duplicateIndex = i;
                }
            }
        }

        if (!isDuplicate) {
            filteredDetections.push(detection);
        } else if (duplicateIndex !== -1 && detection.score > filteredDetections[duplicateIndex].score) {
            filteredDetections[duplicateIndex] = detection;
        }
    }

    return filteredDetections;
}

function calculateIoU(detection1, detection2) {
    const xOverlap = Math.max(0, Math.min(detection1.x2, detection2.x2) - Math.max(detection1.x1, detection2.x1));
    const yOverlap = Math.max(0, Math.min(detection1.y2, detection2.y2) - Math.max(detection1.y1, detection2.y1));
    const overlapArea = xOverlap * yOverlap;

    const area1 = (detection1.x2 - detection1.x1) * (detection1.y2 - detection1.y1);
    const area2 = (detection2.x2 - detection2.x1) * (detection2.y2 - detection2.y1);
    const unionArea = area1 + area2 - overlapArea;

    return overlapArea / unionArea;
}

const filteredResults = removeDuplicates(results, iouThreshold);

// Display results
for (const { x1, x2, y1, y2, score, keypoints } of filteredResults) {
    console.log(`Found person at [${x1}, ${y1}, ${x2}, ${y2}] with score ${score.toFixed(3)}`)
    for (let i = 0; i < keypoints.length; i += 3) {
        const label = model.config.id2label[Math.floor(i / 3)];
        const [x, y, point_score] = keypoints.slice(i, i + 3);
        if (point_score < pointThreshold) continue;
        console.log(`  - ${label}: (${x.toFixed(2)}, ${y.toFixed(2)}) with score ${point_score.toFixed(3)}`);
    }
}

See example output

Found person at [535.7708740234375, 45.77457022666931, 644.4645690917969, 312.20427117347714] with score 0.697
  - nose: (441.61, 87.47) with score 0.966
  - left_eye: (449.36, 79.91) with score 0.988
  - right_eye: (436.36, 79.56) with score 0.850
  - left_ear: (462.02, 83.57) with score 0.919
  - left_shoulder: (478.73, 127.16) with score 0.994
  - right_shoulder: (420.37, 126.47) with score 0.703
  - left_elbow: (503.33, 180.38) with score 0.977
  - left_wrist: (506.53, 236.52) with score 0.924
  - left_hip: (470.67, 223.60) with score 0.982
  - right_hip: (432.32, 223.90) with score 0.851
  - left_knee: (470.86, 306.20) with score 0.949
  - right_knee: (428.56, 306.69) with score 0.601
  - left_ankle: (463.92, 383.59) with score 0.737
Found person at [-0.06377220153808594, 61.59769003391266, 156.24676704406738, 370.5519897222519] with score 0.926
  - nose: (59.61, 100.49) with score 0.979
  - left_eye: (66.44, 96.11) with score 0.954
  - right_eye: (55.82, 96.21) with score 0.908
  - left_ear: (76.90, 98.52) with score 0.819
  - right_ear: (49.82, 102.11) with score 0.571
  - left_shoulder: (87.07, 135.82) with score 0.990
  - right_shoulder: (36.53, 134.96) with score 0.987
  - left_elbow: (102.21, 193.66) with score 0.970
  - right_elbow: (24.85, 187.30) with score 0.947
  - left_wrist: (110.61, 245.75) with score 0.962
  - right_wrist: (6.28, 233.46) with score 0.939
  - left_hip: (82.71, 230.04) with score 0.997
  - right_hip: (48.15, 235.65) with score 0.995
  - left_knee: (95.27, 321.57) with score 0.993
  - right_knee: (52.73, 320.56) with score 0.991
  - left_ankle: (100.90, 415.89) with score 0.948
  - right_ankle: (56.65, 417.09) with score 0.942
Found person at [109.67742919921875, 12.466975402832032, 501.75636291503906, 533.3693368911744] with score 0.934
  - nose: (126.43, 96.98) with score 0.715
  - left_eye: (126.52, 88.36) with score 0.664
  - left_ear: (136.92, 78.79) with score 0.934
  - left_shoulder: (191.69, 125.31) with score 0.998
  - right_shoulder: (166.08, 138.95) with score 0.993
  - left_elbow: (254.38, 194.23) with score 0.997
  - right_elbow: (186.09, 258.25) with score 0.986
  - left_wrist: (309.75, 260.93) with score 0.990
  - right_wrist: (133.20, 283.14) with score 0.973
  - left_hip: (281.07, 280.72) with score 1.000
  - right_hip: (258.20, 300.47) with score 1.000
  - left_knee: (228.48, 442.67) with score 0.999
  - right_knee: (250.90, 474.40) with score 0.999
  - left_ankle: (343.96, 435.26) with score 0.979
  - right_ankle: (340.41, 601.64) with score 0.971
Found person at [422.38683700561523, 67.97338972091676, 638.0375099182129, 493.7016093254089] with score 0.932
  - nose: (417.60, 144.74) with score 0.989
  - left_eye: (426.67, 134.88) with score 0.959
  - right_eye: (410.81, 135.93) with score 0.952
  - left_ear: (443.39, 137.08) with score 0.771
  - right_ear: (400.11, 142.05) with score 0.753
  - left_shoulder: (446.92, 202.43) with score 0.997
  - right_shoulder: (374.31, 196.36) with score 0.993
  - left_elbow: (458.77, 287.40) with score 0.990
  - right_elbow: (355.46, 260.60) with score 0.971
  - left_wrist: (488.87, 354.68) with score 0.984
  - right_wrist: (402.03, 263.57) with score 0.978
  - left_hip: (432.69, 349.58) with score 0.998
  - right_hip: (381.51, 366.30) with score 0.996
  - left_knee: (463.97, 447.94) with score 0.991
  - right_knee: (403.90, 511.95) with score 0.978
  - left_ankle: (450.14, 562.29) with score 0.889
  - right_ankle: (436.81, 548.29) with score 0.759