Building an Enterprise Football Data Pipeline: Decoding Flashscore's Protocol for xG & Referee Analytics

[ League & Season Selection ]
            │
            ▼
┌───────────────────────────────────────────┐
│ Phase 1: Browser Handshake (Puppeteer)    │
│  - Captures x-fsign security tokens       │
│  - Extracts countryId & tourId            │
└─────────────────────┬─────────────────────┘
                      │
           [ Immediate Browser Shutdown ]
            (RAM drops from 1.2GB -> 70MB)
                      │
                      ▼
┌───────────────────────────────────────────┐
│ Phase 2: Parallel HTTP/2 Feed Workers     │
│  - got-scraping with JA3 TLS matching     │
│  - Decodes df_st_1_ (Stats) & df_sui_1_   │
└─────────────────────┬─────────────────────┘
                      │
                      ▼
┌───────────────────────────────────────────┐
│ Self-Healing Recovery Pass                │
│  - Auto-retries skipped/failed matches    │
└─────────────────────┬─────────────────────┘
                      │
                      ▼
┌───────────────────────────────────────────┐
│ Dataset Output (JSON / Pandas / Excel)    │
└───────────────────────────────────────────┘

// src/main.ts (Handshake & Network Interception)

const onPageCreated = (page: Page) => {
    const listener = (request: HTTPRequest) => {
        const url = request.url();
        // Intercept dynamic background API calls (e.g. tr_1_countryId_tourId)
        const match = url.match(/_1_(\d+)_([A-Za-z0-9]{8})/);
        if (match) {
            countryId = match[1];
            tourId = match[2];
            page.off('request', listener);
            networkResolver(); // Instantly resolve network promise
        }
    };
    page.on('request', listener);
};

const { context, browser, page } = await SessionManager.startDiscovery(discoveryUrl, onPageCreated);

// Perform match discovery...
const matchesMetadata = await DiscoveryManager.discoverMatches(page, input.targetCount, context, tourId, countryId);

// CRITICAL: Close the browser immediately to save compute memory
console.log('[Main] Discovery complete. Closing browser...');
await browser.close();

SF÷Top stats~SG÷Expected goals (xG)¬SH÷1.42¬SI÷0.88~SG÷Ball possession¬SH÷58%¬SI÷42%~

// src/parser.ts

export class FlashscoreParser {
    private static KEY_MAP: Record<string, string> = {
        'Expected goals (xG)': 'expected_goals',
        'Ball possession': 'ball_possession',
        'Total shots': 'total_shots',
        'Shots on target': 'shots_on_target',
        'Corner kicks': 'corner_kicks',
        'Passes': 'passes',
        'Yellow cards': 'yellow_cards',
        'Fouls': 'fouls',
        'Offsides': 'offsides'
    };

    static parseRow(row: string): Record<string, string> {
        const obj: Record<string, string> = {};
        const cleanRow = row.startsWith('~') ? row.substring(1) : row;
        const fields = cleanRow.split('¬');

        for (const field of fields) {
            if (!field.includes('÷')) continue;
            const [key, value] = field.split('÷');
            if (key) obj[key] = value ?? '';
        }
        return obj;
    }

    static parseStatistics(raw: string): FlatMatchStatistics {
        const records = raw.split('~');
        const stages: any[] = [];
        let currentStage: any = null;

        for (const record of records) {
            const data = this.parseRow(record);
            if (record.startsWith('SF÷') && data['SF'] === 'Top stats') {
                currentStage = {};
                stages.push(currentStage);
                continue;
            }

            if ((record.startsWith('SD÷') || record.startsWith('SG÷')) && currentStage) {
                const rawName = data['SG'] || '';
                const key = this.KEY_MAP[rawName] || rawName.toLowerCase().replace(/\s+/g, '_');

                if (!currentStage[key]) {
                    currentStage[key] = {
                        home: this.sanitizeValue(data['SH']),
                        away: this.sanitizeValue(data['SI'])
                    };
                }
            }
        }

        return {
            fullTime: stages[0] || {},
            firstHalf: stages[1] || {},
            secondHalf: stages[2] || {}
        };
    }
}

{
  "minute": "32'",
  "type": "Goal",
  "team": "home",
  "scorer": {
    "name": "Maldini D.",
    "url": "/player/maldini-daniel/txSCnXec/",
    "id": "txSCnXec"
  },
  "assist": {
    "name": "Bellanova R.",
    "url": "/player/bellanova-raoul/6BgYlqqU/",
    "id": "6BgYlqqU"
  }
}

// src/main.ts (Self-Healing Recovery)

if (failedMatches.length > 0) {
    console.log(`[Main] 🔄 Self-Healing Final Pass: Retrying ${failedMatches.length} skipped match(es)...`);
    await new Promise(r => setTimeout(r, 3000));

    for (const meta of failedMatches) {
        try {
            const statsPath = `df_st_1_${meta.id}`;
            const rawStats = await Scraper.getRawFeed(statsPath, context);
            const flatStats = FlashscoreParser.parseStatistics(rawStats);
            // ... process and recover match record ...
            await Actor.pushData(fullMatch);
            console.log(`[Main] 🚀 Successfully recovered match: ${meta.homeTeam} vs ${meta.awayTeam}`);
        } catch (recoveryErr: any) {
            console.error(`[Main] Recovery failed for ${meta.id}: ${recoveryErr.message}`);
        }
    }
}

import pandas as pd

# Fetch dataset directly from Apify JSON API endpoint
dataset_url = "https://api.apify.com/v2/key-value-stores/f1pLKNreZAd2R6acn/records/serie-a-2024-25-full.json?disableRedirect=true"
df = pd.read_json(dataset_url)

# Inspect Expected Goals (xG), Referee, and Scores
print(df[['homeTeam', 'awayTeam', 'referee', 'halfTimeScore', 'fullTimeScore', 'homeXg', 'awayXg']].head())