Breaking News: Grepper is joining You.com. Read the official announcement!

pyspark dataframe machine learning

Innocent Iguana answered on February 17, 2024 Popularity 6/10 Helpfulness 6/10

answer pyspark dataframe machine learning

pyspark dataframe machine learning

Comment

# One-hot encoding
from pyspark.ml.feature import StringIndexer, OneHotEncoder
# StringIndexer does indexing for each category. this step allows handling unseen category in testing set
string_indexer1 = StringIndexer(inputCol="cat_col",outputCol="string_index") 
one_hot_encoder1 = OneHotEncoder(inputCol="string_index",outputCol="onehot_feature") # One-hot encoding using the category indices
# Combine all features
from pyspark.ml.feature import VectorAssembler
vec_assembler = VectorAssembler(inputCols=["feature1", "feature2", "feature3", "onehot_feature1", "onehot_feature2"], outputCol="features")

# Define the model
model_rf = RandomForestClassifier(featuresCol="features", labelCol="label", numTrees=10) # from pyspark.ml.classification import RandomForestClassifier
model_lr1 = LogisticRegression(featuresCol="features", labelCol="label") # from pyspark.ml.classification import LogisticRegression
# elasticNetParam = 0 is ridge regression, elasticNetParam = 1 is lasso regression (regularization parameters)
model_lr2 = LinearRegression(featuresCol="features", labelCol="label", elasticNetParam=0, regParam=0.1) # from pyspark.ml.regression import LinearRegression
model_kmeans = KMeans(featuresCol="features", predictionCol="kmeans_prediction", k=3) # from pyspark.ml.clustering import KMeans
# deep learninng
layers = [len(feature_cols) + 2, 5, 2]  # Input layer size, hidden layer sizes, output layer size
model_dl = MultilayerPerceptronClassifier(layers=layers, labelCol="label", featuresCol="features", seed=123)
# Pipeline
from pyspark.ml import Pipeline
pipeline = Pipeline(stages=[string_indexer1, one_hot_encoder1, string_indexer2, one_hot_encoder2, vec_assembler, model_xx])

# Create the parameter grid
import pyspark.ml.tuning as tune
paramGrid = tune.ParamGridBuilder()\
        .addGrid(lr_model.regParam, np.arange(0, .1, .01))
        .addGrid(lr_model.elasticNetParam, [0, 1])
        .build()

# Evaluation metric
import pyspark.ml.evaluation as evals
evaluator_logistic = evals.BinaryClassificationEvaluator(labelCol="label", rawPredictionCol="rawPrediction")
evaluator_reg = evals.RegressionEvaluator(labelCol="label", predictionCol="prediction", metricName="rmse")
evaluator_rf_dl = evals.MulticlassClassificationEvaluator(labelCol="label", predictionCol="prediction", metricName="accuracy") 
# Create cross-validator
cv = CrossValidator(estimator=pipeline,
                          estimatorParamMaps=paramGrid,
                          evaluator=evaluator,
                          numFolds=3, seed = 42) 

# Split the data into training and test sets
training, test = transformed_df.randomSplit([.6, .4])
total_models = len(paramGrid)
cvModel = cv.fit(training) # Fit the dataframe
bestModel = cvModel.bestModel # Best model
bestModel.stages # See stages
bestModel.stages[3].extractParamMap() # See parameters in the model, model is on index 3 of pipeline
bestParams = bestModel.stages[-1].extractParamMap() # See best parameters
test_results = bestModel.transform(test) # Use the model to predict the test set
predictions = cvModel.transform(test) # Predict using testing set
accuracy = evaluator.evaluate(predictions)
test_results.groupBy("label", "prediction").count().show() # Confusion matrix
feature_importances = bestModel.stages[-1].featureImportances

Popularity 6/10 Helpfulness 6/10 Language python

Source: Grepper

Tags: machine-learning python

Link to this answer
Share Copy Link

Contributed on Feb 17 2024

Innocent Iguana

0 Answers Avg Quality 2/10

pyspark dataframe machine learning

Contents

More Related Answers

pyspark dataframe machine learning

Grepper

Documentation

Social

Legal

Contact

Oops, You will need to install Grepper and log-in to perform this action.