[docs]defload_data(path,index,target,n_rows=None,drop=None,verbose=True,**kwargs):"""Load features and target from file. Args: path (str): Path to file or a http/ftp/s3 URL. index (str): Column for index. target (str): Column for target. n_rows (int): Number of rows to return. Defaults to None. drop (list): List of columns to drop. Defaults to None. verbose (bool): If True, prints information about features and target. Defaults to True. **kwargs: Other keyword arguments that should be passed to panda's `read_csv` method. Returns: pd.DataFrame, pd.Series: Features matrix and target. """feature_matrix=pd.read_csv(path,index_col=index,nrows=n_rows,**kwargs)targets=[target]+(dropor[])y=feature_matrix[target]X=feature_matrix.drop(columns=targets)ifverbose:# number of featuresprint(number_of_features(X.dtypes),end="\n\n")# number of total training examplesinfo="Number of training examples: {}"print(info.format(len(X)),end="\n")# target distributionprint(target_distribution(y))returninfer_feature_types(X),infer_feature_types(y)
[docs]defsplit_multiseries_data(X,y,series_id,time_index,**kwargs):"""Split stacked multiseries data into train and test sets. Unstacked data can use `split_data`. Args: X (pd.DataFrame): The input training data of shape [n_samples*n_series, n_features]. y (pd.Series): The target training targets of length [n_samples*n_series]. series_id (str): Name of column containing series id. time_index (str): Name of column containing time index. **kwargs: Additional keyword arguments to pass to the split_data function. Returns: pd.DataFrame, pd.DataFrame, pd.Series, pd.Series: Feature and target data each split into train and test sets. """X_unstacked,y_unstacked=unstack_multiseries(X,y,series_id,time_index,y.name,)(X_train_unstacked,X_holdout_unstacked,y_train_unstacked,y_holdout_unstacked,)=split_data(X_unstacked,y_unstacked,problem_type="time series regression",**kwargs)# Get unique series values (as a list to maintain order) from X if there is only the time_index column# Otherwise, this information is generated in `stack_X` from the column valuesseries_id_values=X[series_id].unique()iflen(X_unstacked.columns)==1elseNoneX_train=stack_X(X_train_unstacked,series_id,time_index,series_id_values=series_id_values,)X_holdout=stack_X(X_holdout_unstacked,series_id,time_index,starting_index=X_train.index[-1]+1,series_id_values=series_id_values,)y_train=stack_data(y_train_unstacked)y_holdout=stack_data(y_holdout_unstacked,starting_index=y_train.index[-1]+1)returnX_train,X_holdout,y_train,y_holdout
[docs]defsplit_data(X,y,problem_type,problem_configuration=None,test_size=None,random_seed=0,):"""Split data into train and test sets. Args: X (pd.DataFrame or np.ndarray): data of shape [n_samples, n_features] y (pd.Series, or np.ndarray): target data of length [n_samples] problem_type (str or ProblemTypes): type of supervised learning problem. see evalml.problem_types.problemtype.all_problem_types for a full list. problem_configuration (dict): Additional parameters needed to configure the search. For example, in time series problems, values should be passed in for the time_index, gap, and max_delay variables. test_size (float): What percentage of data points should be included in the test set. Defaults to 0.2 (20%) for non-timeseries problems and 0.1 (10%) for timeseries problems. random_seed (int): Seed for the random number generator. Defaults to 0. Returns: pd.DataFrame, pd.DataFrame, pd.Series, pd.Series: Feature and target data each split into train and test sets. Raises: ValueError: If the problem_configuration is missing or does not contain both a time_index and series_id for multiseries problems. Examples: >>> X = pd.DataFrame([1, 2, 3, 4, 5, 6], columns=["First"]) >>> y = pd.Series([8, 9, 10, 11, 12, 13]) ... >>> X_train, X_validation, y_train, y_validation = split_data(X, y, "regression", random_seed=42) >>> X_train First 5 6 2 3 4 5 3 4 >>> X_validation First 0 1 1 2 >>> y_train 5 13 2 10 4 12 3 11 dtype: int64 >>> y_validation 0 8 1 9 dtype: int64 """ifis_multiseries(problem_type)andisinstance(y,pd.Series):ifproblem_configurationisNone:raiseValueError("split_data requires problem_configuration for multiseries problems",)series_id=problem_configuration.get("series_id")time_index=problem_configuration.get("time_index")ifseries_idisNoneortime_indexisNone:raiseValueError("split_data needs both series_id and time_index values in the problem_configuration to split multiseries data",)returnsplit_multiseries_data(X,y,series_id,time_index,problem_configuration=problem_configuration,test_size=test_size,random_seed=random_seed,)X=infer_feature_types(X)y=infer_feature_types(y)data_splitter=Noneifis_time_series(problem_type):iftest_sizeisNone:test_size=0.1if(problem_configurationisnotNoneand"forecast_horizon"inproblem_configuration):fh_pct=problem_configuration["forecast_horizon"]/len(X)test_size=max(test_size,fh_pct)data_splitter=TrainingValidationSplit(test_size=test_size,shuffle=False,stratify=None,random_seed=random_seed,)else:iftest_sizeisNone:test_size=0.2ifis_regression(problem_type):data_splitter=ShuffleSplit(n_splits=1,test_size=test_size,random_state=random_seed,)elifis_classification(problem_type):data_splitter=StratifiedShuffleSplit(n_splits=1,test_size=test_size,random_state=random_seed,)train,test=next(data_splitter.split(X,y))X_train=X.ww.iloc[train]X_test=X.ww.iloc[test]y_train=y.ww.iloc[train]y_test=y.ww.iloc[test]returnX_train,X_test,y_train,y_test
[docs]defnumber_of_features(dtypes):"""Get the number of features of each specific dtype in a DataFrame. Args: dtypes (pd.Series): DataFrame.dtypes to get the number of features for. Returns: pd.Series: dtypes and the number of features for each input type. Example: >>> X = pd.DataFrame() >>> X["integers"] = [i for i in range(10)] >>> X["floats"] = [float(i) for i in range(10)] >>> X["strings"] = [str(i) for i in range(10)] >>> X["booleans"] = [bool(i%2) for i in range(10)] Lists the number of columns corresponding to each dtype. >>> number_of_features(X.dtypes) Number of Features Boolean 1 Categorical 1 Numeric 2 """dtype_to_vtype={"bool":"Boolean","int32":"Numeric","int64":"Numeric","float64":"Numeric","object":"Categorical","datetime64[ns]":"Datetime",}vtypes=dtypes.astype(str).map(dtype_to_vtype).value_counts()returnvtypes.sort_index().to_frame("Number of Features")
[docs]deftarget_distribution(targets):"""Get the target distributions. Args: targets (pd.Series): Target data. Returns: pd.Series: Target data and their frequency distribution as percentages. Examples: >>> y = pd.Series([1, 2, 4, 1, 3, 3, 1, 2]) >>> print(target_distribution(y).to_string()) Targets 1 37.50% 2 25.00% 3 25.00% 4 12.50% >>> y = pd.Series([True, False, False, False, True]) >>> print(target_distribution(y).to_string()) Targets False 60.00% True 40.00% """distribution=targets.value_counts()/len(targets)returndistribution.mul(100).apply("{:.2f}%".format).rename_axis("Targets")