import pandas as pd
import json

with open("food101/metadata.json") as f:
    data = json.load(f)


# This is the dataset level information
info_df = pd.DataFrame([data["info"]])

# This is the image level information
media_items_df = pd.json_normalize(data["media_items"])

info_df

media_items_df

UNIQUENESS_SCORE_THRESHOLD = 0.85

coreset_df = media_items_df[
    media_items_df["uniqueness_score"] > UNIQUENESS_SCORE_THRESHOLD
]
coreset_df

def has_duplicate_issue(metadata_items, confidence_threshold=0.8):
    if not isinstance(metadata_items, list):
        return False

    for item in metadata_items:
        if (
            item.get("type") == "issue"
            and item.get("properties", {}).get("issue_type") == "duplicates"
            and item.get("properties", {}).get("confidence", 0) > confidence_threshold
        ):
            return True
    return False


# Replace with your confidence threshold
CONFIDENCE_THRESHOLD = 0.8

duplicate_df = media_items_df[
    media_items_df["metadata_items"].apply(
        lambda x: has_duplicate_issue(x, confidence_threshold=CONFIDENCE_THRESHOLD)
    )
]

duplicate_df

def has_mislabel_issue(metadata_items, confidence_threshold=0.8):
    if not isinstance(metadata_items, list):
        return False

    for item in metadata_items:
        if (
            item.get("type") == "issue"
            and item.get("properties", {}).get("issue_type") == "mislabels"
            and item.get("properties", {}).get("confidence", 0) > confidence_threshold
        ):
            return True
    return False


# Replace with your confidence threshold
CONFIDENCE_THRESHOLD = 0.8

mislabel_df = media_items_df[
    media_items_df["metadata_items"].apply(
        lambda x: has_mislabel_issue(x, confidence_threshold=CONFIDENCE_THRESHOLD)
    )
]


mislabel_df