RecoMetrics

https://recometrics.readthedocs.io/en/latest
Admin
RecoMetrics is a library-agnostic evaluation framework for implicit-feedback recommender systems...
Tokens:6,991
Snippets:53
Trust Score:9.6
Update:1 month ago
Show doc for...
Tokens:
Raw
### Install RecoMetrics via pip

Source: https://recometrics.readthedocs.io/en/latest/_sources/index.rst.txt

Use this command to install the package from PyPI.

```bash
pip install recometrics
```

--------------------------------

### Calculate Recommendation Metrics

Source: https://recometrics.readthedocs.io/en/latest/_modules/recometrics.html

Internal logic for validating inputs, preparing data structures, and invoking C++ functions to compute various recommendation metrics.

```python
    if (nthreads > 1) and (not cpp_funs._get_has_openmp()):
        msg_omp  = "Attempting to use more than 1 thread, but "
        msg_omp += "package was built without multi-threading "
        msg_omp += "support - see the project's GitHub page for "
        msg_omp += "more information."
        warn(msg_omp)

    if k > X_test.shape[1]:
        raise ValueError("'k' should be smaller than the number of items.")

    if item_biases is not None:
        assert isinstance(item_biases, np.ndarray)
        if (len(item_biases.shape) > 2):
            raise ValueError("'item_biases' should be a 1-d array.")
        if (len(item_biases.shape) != 1):
            item_biases = item_biases.reshape(-1)
        if (not item_biases.shape[0]):
            raise ValueError("'item_biases' is empty.")
        if item_biases.shape[0] < X_test.shape[1]:
            raise ValueError("Number of items in 'item_biases' must match with 'X_test'.")
        if item_biases.shape[0] > X_test.shape[1]:
            item_biases = item_biases[:X_test.shape[1]]
            warn("'item_biases' has more items than 'X_test'.")
        
        item_biases = _cast_to_dtype(item_biases, use_float)
        A = np.c_[A, np.ones((A.shape[0], 1), dtype=ctypes.c_double if not use_float else ctypes.c_float)]
        B = np.c_[B, item_biases.reshape((-1,1))]
        item_biases = None

    X_train = _as_csr(X_train)
    X_test = _as_csr(X_test)
    
    X_train = _cast_indices_to_int32(X_train)
    X_test = _cast_to_dtype(X_test, use_float)
    X_test = _cast_indices_to_int32(X_test)
    A = _cast_to_dtype(A, use_float)
    A, lda = _as_row_major(A)
    B = _cast_to_dtype(B, use_float)
    B, ldb = _as_row_major(B)

    p_at_k, tp_at_k, r_at_k, ap_at_k, tap_at_k, ndcg_at_k, \
    hit_at_k, rr_at_k, roc_auc, pr_auc = \
        cpp_funs.calc_reco_metrics(
            A, lda,
            B, ldb,
            X_train, X_test,
            k_metrics = k,
            precision = precision,
            trunc_precision = trunc_precision,
            recall = recall,
            average_precision = average_precision,
            trunc_average_precision = trunc_average_precision,
            ndcg = ndcg,
            hit = hit,
            rr = rr,
            roc_auc = roc_auc,
            pr_auc = pr_auc,
            break_ties_with_noise = break_ties_with_noise,
            min_pos_test = min_pos_test,
            min_items_pool = min_items_pool,
            consider_cold_start = consider_cold_start,
            cumulative = cumulative,
            nthreads = nthreads,
            seed = seed
        )

    out = dict()
    if p_at_k.shape[0]:
        out["P@K"] = p_at_k
    if tp_at_k.shape[0]:
        out["TP@K"] = tp_at_k
    if r_at_k.shape[0]:
        out["R@K"] = r_at_k
    if ap_at_k.shape[0]:
        out["AP@K"] = ap_at_k
    if tap_at_k.shape[0]:
        out["TAP@K"] = tap_at_k
    if ndcg_at_k.shape[0]:
        out["NDCG@K"] = ndcg_at_k
    if hit_at_k.shape[0]:
        out["Hit@K"] = hit_at_k
    if rr_at_k.shape[0]:
        out["RR@K"] = rr_at_k
    if roc_auc.shape[0]:
        out["ROC_AUC"] = roc_auc
    if pr_auc.shape[0]:
        out["PR_AUC"] = pr_auc

    if not as_df:
        out["K"] = k

    if as_df:
        if not cumulative:
            out = pd.DataFrame(out)
            if rename_k:
                out.columns = out.columns.str.replace("@K$", "@"+str(k), regex=True)
        else:
            out = [
                pd.DataFrame(v, columns=[re.sub("(@)K$", r"\1", k)+(str(i+1) if bool(re.search("@K$", k)) else "")
                                         for i in range(v.shape[1])])
                for k,v in out.items()
            ]
            out = pd.concat(out, axis=1)

    return out
```

--------------------------------

### Metric and Input Validation Logic

Source: https://recometrics.readthedocs.io/en/latest/_modules/recometrics.html

Handles parameter normalization, input matrix validation, and metric flag configuration for recommendation evaluation.

```python
    if all_metrics:
        precision = True
        trunc_precision = True
        recall = True
        average_precision = True
        trunc_average_precision = True
        ndcg = True
        hit = True
        rr = True
        roc_auc = True
        pr_auc = True

    if (item_biases is not None) and (isinstance(item_biases, pd.Series)):
        item_biases = item_biases.to_numpy()

    if (A is None) != (B is None):
        raise ValueError("'A' and 'B' must either be passed together or passed as 'None' together.")
    if (A is None):
        if (item_biases is None):
            raise ValueError("Must pass item biases if not passing factors.")
        A = np.ones((X_test.shape[0],1), dtype=ctypes.c_double, order="C")
        B = np.ascontiguousarray(item_biases, dtype=ctypes.c_double).reshape((-1,1))
        item_biases = None

    assert isinstance(A, np.ndarray)
    assert isinstance(B, np.ndarray)
    assert issparse(X_test)

    if X_test.shape[0] >= np.iinfo(np.int32).max:
        raise ValueError("Number of test user is larger than maximum supported.")
    if X_test.shape[1] >= np.iinfo(np.int32).max:
        raise ValueError("Number of items is larger than maximum supported.")
    if not X_test.data.shape[0]:
        raise ValueError("'X_test' is empty.")

    if len(A.shape) != 2:
        raise ValueError("'A' must be a 2-dimensional array.")
    if len(B.shape) != 2:
        raise ValueError("'B' must be a 2-dimensional array.")
    if A.shape[1] != B.shape[1]:
        raise ValueError("'A' and 'B' must have the same number of columns.")
    if (not A.shape[0]) or (not A.shape[1]) or (not B.shape[1]) or (not X_test.shape[0]) or (not X_test.shape[1]):
        raise ValueError("Input matrices cannot be empty.")
    if A.shape[0] < X_test.shape[0]:
        raise ValueError("Number of users in 'A' and 'X_test' does not match.")
    if B.shape[0] < X_test.shape[1]:
        raise ValueError("Number of items in 'B' and 'X_test' does not match.")

    if (A.shape[0] > X_test.shape[0]):
        warn("'A' has more users than 'X_test'.")
        A = A[:X_test.shape[0], :]
    if (B.shape[0] > X_test.shape[1]):
        warn("'B' has more items than 'X_test'.")
        B = B[:X_test.shape[1], :]

    use_float = (A.dtype == ctypes.c_float) and (B.dtype == ctypes.c_float)

    if X_train is None:
        X_train = csr_array(X_test.shape, dtype=ctypes.c_double if not use_float else ctypes.c_float)
        consider_cold_start = True

    assert issparse(X_train)
    assert X_train.shape[1] == X_test.shape[1]
    if (X_train.shape[0] < X_test.shape[0]):
        raise ValueError("'X_train' and 'X_test' should have the same number of rows.")
    elif (X_train.shape[0] > X_test.shape[0]):
        warn("'X_train' mas more rows than 'X_test'.")

    as_df = bool(as_df)
    break_ties_with_noise = bool(break_ties_with_noise)
    consider_cold_start = bool(consider_cold_start)
    cumulative = bool(cumulative)
    rename_k = bool(rename_k)
    
    precision = bool(precision)
    trunc_precision = bool(trunc_precision)
    recall = bool(recall)
    average_precision = bool(average_precision)
    trunc_average_precision = bool(trunc_average_precision)
    ndcg = bool(ndcg)
    hit = bool(hit)
    rr = bool(rr)
    roc_auc = bool(roc_auc)
    pr_auc = bool(pr_auc)
    
    if (not precision) and (not average_precision) and (not ndcg) and (not hit) and (not rr) and (not roc_auc):
        raise ValueError("Must pass at least one metric to calculate.")

    if isinstance(seed, np.random.RandomState):
        seed = int(seed.randint(np.iinfo(np.int32).max))
    elif isinstance(seed, np.random.Generator):
        seed = int(seed.integers(np.iinfo(np.int32).max))

    nthreads = int(nthreads)
    seed = int(seed)
    k = int(k)
    min_pos_test = int(min_pos_test)
    min_items_pool = int(min_items_pool)

    assert seed >= 1
    assert k >= 1
    assert min_pos_test >= 1
    assert min_items_pool >= 1

    if nthreads is None:
        nthreads = 1
    elif nthreads < 0:
        nthreads = multiprocessing.cpu_count() + 1 + nthreads
    assert nthreads > 0
```

--------------------------------

### Calculate Recommendation Quality Metrics

Source: https://recometrics.readthedocs.io/en/latest/_modules/recometrics.html

Computes metrics like P@K, TP@K, and R@K for implicit-feedback systems based on user and item factors.

```python
def calc_reco_metrics(
    X_train, X_test,
    A, B,
    k = 5,
    item_biases = None,
    as_df = True,
    precision = True,
    trunc_precision = False,
    recall = False,
    average_precision = True,
    trunc_average_precision = False,
    ndcg = True,
    hit = False,
    rr = False,
    roc_auc = False,
    pr_auc = False,
    all_metrics = False,
    rename_k = True,
    break_ties_with_noise = True,
    min_pos_test = 1,
    min_items_pool = 2,
    consider_cold_start = True,
    cumulative = False,
    nthreads = -1,
    seed = 1
):
```

--------------------------------

### Validate and prepare data for splitting

Source: https://recometrics.readthedocs.io/en/latest/_modules/recometrics.html

Validates input parameters and ensures the input matrix is in CSR format before performing data splits.

```python
if (max_test_users is None) or (max_test_users == 0):
        max_test_users = X.shape[0]
    
    assert max_test_users > 0
    assert seed >= 0
    assert min_pos_test >= 0
    assert min_items_pool >= 0
    
    max_test_users = int(max_test_users)
    seed = int(seed)
    min_pos_test = int(min_pos_test)
    min_items_pool = int(min_items_pool)
    
    if users_test_fraction is not None:
        assert (users_test_fraction > 0) and (users_test_fraction < 1)
        users_test_fraction = float(users_test_fraction)

    assert (items_test_fraction > 0) and (items_test_fraction < 1)
    items_test_fraction = float(items_test_fraction)

    assert split_type in ("all", "separated", "joined")

    consider_cold_start = bool(consider_cold_start)


    if min_pos_test >= X.shape[1]:
        raise ValueError("'min_pos_test' must be smaller than the number of columns in 'X'.")

    if min_items_pool >= X.shape[1]:
        raise ValueError("'min_items_pool' must be smaller than the number of columns in 'X'.")

    if split_type != "all":
        if X.shape[0] < 2:
            raise ValueError("'X' has less than 2 rows.")
        if users_test_fraction is not None:
            n_users_take = X.shape[0] * users_test_fraction
            if n_users_take < 1:
                warn("Desired fraction of test users implies <1, will select 1 user.")
                n_users_take = 1
            n_users_take = round(n_users_take)
            n_users_take = min(n_users_take, max_test_users)
        else:
            if max_test_users > X.shape[0]:
                warn("'max_test_users' is larger than number of users. Will take all.")
            n_users_take = min(max_test_users, X.shape[0])

    X = _as_csr(X)
    if (not X.shape[0]) or (not X.shape[1]):
        raise ValueError("'X' cannot be empty.")
    if X.dtype not in (np.float32, np.float64):
        X = X.astype(np.float64)
    if not X.data.shape[0]:
        raise ValueError("'X' contains no non-zero entries.")
    X = _cast_indices_to_int32(X)
```

--------------------------------

### split_data

Source: https://recometrics.readthedocs.io/en/latest/index.html

Splits implicit feedback data into training and testing sets based on specified fractions and constraints.

```APIDOC
## Function: split_data(X, split_type, users_test_fraction, max_test_users, items_test_fraction, min_items_pool, min_pos_test, consider_cold_start, seed)

### Description
Splits an implicit feedback matrix (CSR) into training and testing components for recommender system evaluation. Supports different split strategies including 'all', 'separated', and 'joined'.

### Parameters
- **X** (CSR matrix) - Required - The implicit feedback data (users as rows, items as columns).
- **split_type** (str) - Required - Type of split: 'all', 'separated', or 'joined'.
- **users_test_fraction** (float or None) - Optional - Target fraction of users to set as test.
- **max_test_users** (float or None) - Optional - Maximum number of users to set as test.
- **items_test_fraction** (float) - Required - Target fraction of items to set as test for each user.
- **min_items_pool** (int) - Required - Minimum items a user must have to be eligible for testing.
- **min_pos_test** (int) - Required - Minimum positive entries a user must have to be eligible for testing.
- **consider_cold_start** (bool) - Required - Whether to allow users with no positive training data in the test set.
- **seed** (int) - Required - Random seed for reproducibility.

### Returns
- **split** (tuple) - A tuple containing the resulting sparse matrices (X_train, X_test, and optionally X_rem and users_test) based on the chosen split_type.
```

--------------------------------

### recometrics.calc_reco_metrics

Source: https://recometrics.readthedocs.io/en/latest/index.html

Calculates recommendation quality metrics for implicit-feedback recommender systems based on user and item factors.

```APIDOC
## Function: recometrics.calc_reco_metrics

### Description
Calculates recommendation quality metrics (P@K, R@K, AP@K, etc.) for implicit-feedback recommender systems based on low-rank matrix factorization. Metrics are calculated on a per-user basis by ranking items according to model predictions while ignoring items present in the training data.

### Parameters
- **X_train** (array-like) - Training data interactions.
- **X_test** (array-like) - Test data interactions.
- **A** (array-like) - User factors matrix.
- **B** (array-like) - Item factors matrix.
- **k** (int, default=5) - Number of top items to consider for ranking metrics.
- **item_biases** (array-like, optional) - Item biases for prediction adjustment.
- **as_df** (bool, default=True) - Whether to return results as a pandas DataFrame.
- **precision** (bool, default=True) - Calculate Precision@K.
- **recall** (bool, default=False) - Calculate Recall@K.
- **average_precision** (bool, default=True) - Calculate Average Precision@K.
- **nthreads** (int, default=-1) - Number of threads to use for computation.
```

--------------------------------

### recometrics.split_reco_train_test

Source: https://recometrics.readthedocs.io/en/latest/index.html

Creates train-test splits of implicit-feedback data for fitting and evaluating recommender system models.

```APIDOC
## recometrics.split_reco_train_test

### Description
Creates train-test splits of implicit-feedback data (recorded user-item interactions) for fitting and evaluating models for recommender systems. It supports three modes of splitting: 'all', 'separated', and 'joined'.

### Parameters
- **X** (array-like) - Required - The input data containing user-item interactions.
- **split_type** (str) - Optional - The mode of splitting: 'separated' (default), 'all', or 'joined'.
- **users_test_fraction** (float) - Optional - The fraction of users to include in the test set (default: 0.1).
- **max_test_users** (int) - Optional - Maximum number of users to include in the test set (default: 10000).
- **items_test_fraction** (float) - Optional - The fraction of items to include in the test set for each user (default: 0.3).
- **min_items_pool** (int) - Optional - Minimum number of items required in the pool (default: 2).
- **min_pos_test** (int) - Optional - Minimum number of positive interactions required in the test set (default: 1).
- **consider_cold_start** (bool) - Optional - Whether to consider cold start users (default: False).
- **seed** (int) - Optional - Random seed for reproducibility (default: 1).
```

--------------------------------

### Split Implicit-Feedback Data

Source: https://recometrics.readthedocs.io/en/latest/_modules/recometrics.html

Function signature for creating train-test splits from implicit-feedback data.

```python
[docs]
def split_reco_train_test(
    X,
    split_type = "separated",
    users_test_fraction = 0.1,
    max_test_users = 10000,
    items_test_fraction = 0.3,
    min_items_pool = 2,
    min_pos_test = 1,
    consider_cold_start = False,
    seed = 1
):
```

--------------------------------

### Internal Data Processing Utilities

Source: https://recometrics.readthedocs.io/en/latest/_modules/recometrics.html

Helper functions for ensuring data structures are in the correct format, dtype, and memory layout for C-level processing.

```python
def _as_row_major(X):
    if X.flags["C_CONTIGUOUS"]:
        return X, X.shape[1]
    if X.strides[1] != X.dtype.itemsize:
        return np.ascontiguousarray(X, dtype=X.dtype), X.shape[1]
    return X, int(X.strides[0] / X.itemsize)

def _cast_to_dtype(X, use_float):
    if not use_float:
        if X.dtype != ctypes.c_double:
            X = X.astype(ctypes.c_double)
    else:
        if X.dtype != ctypes.c_float:
            X = X.astype(ctypes.c_float)
    return X

def _cast_indices_to_int32(X):
    if (X.indptr.dtype != np.int32) or (X.indices.dtype != np.int32):
        X = X.copy()
        X.indptr = X.indptr.astype(np.int32)
        X.indices = X.indices.astype(np.int32)
    return X

def _as_csr(X):
    if issparse(X):
        if X.format != "csr":
            X = X.tocsr()
        X.sort_indices()
        return X
    else:
        return csr_array(X)
```

--------------------------------

### Split CSR Matrix Logic

Source: https://recometrics.readthedocs.io/en/latest/_modules/recometrics.html

Handles different CSR matrix splitting strategies based on the provided split_type parameter.

```python
if split_type == "all":
        return cpp_funs.split_csr_selected_users(X, items_test_fraction, seed)
    elif split_type == "separated":
        return cpp_funs.split_csr_separated_users(
            X,
            n_users_take,
            items_test_fraction,
            consider_cold_start,
            min_items_pool,
            min_pos_test,
            True,
            seed
        )
    elif split_type == "joined":
        return cpp_funs.split_csr_separated_users(
            X,
            n_users_take,
            items_test_fraction,
            consider_cold_start,
            min_items_pool,
            min_pos_test,
            False,
            seed
        )
    else:
        raise ValueError("Unexpected error.")
```
Tokens:
Raw