如何在 amazon sageMaker 中部署自定义模型

Question

我是 AWS sagemaker 的新手，我正在尝试在 sagemaker 中部署时间序列自定义 lstm 模型，请帮助我以及如何配置脚本模式。 这是我的脚本文件 timer_series.py 代码。

import sagemaker
import boto3
import os
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import tensorflow 
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
from tensorflow.keras.layers import LSTM
from sklearn.preprocessing import MinMaxScaler
from tensorflow.keras.preprocessing.sequence import TimeseriesGenerator

from sklearn.metrics import mean_squared_error


if __name__ =='__main__':

    parser.add_argument('--epochs', type=int, default=50)
    parser.add_argument('--batch_size', type=int, default=72)
    parser.add_argument('--n_train_hours', type=int, default=24*365*2)
    parser.add_argument('--n_validation_hours', type=int, default=24*365*4)

    # input data and model directories
    parser.add_argument('--model_dir', type=str)

    args, _ = parser.parse_known_args()

    train_dataset_dir = os.environ.get('SM_INPUT_DIR') + '/data/training/' 
    output_model_dir = os.environ.get('SM_MODEL_DIR')
    output_object_dir = os.environ.get('SM_OUTPUT_DATA_DIR')

    epochs = args.epochs
    batch_size = args.batch_size
    input_data = {args.input_data}
    dataset = read_csv( train_dataset_dir + 'dataset.csv', header=0, index_col='Date')
    dataset.sort_index(inplace=True)
    train = dataset.iloc[:109]
    test= dataset.iloc[109:]  
    scaler = MinMaxScaler()
    scaled_train = scaler.fit_transform(train)
    scaled_test=scaler.fit_transform(test)
    n_input = 12
    n_feature = 1

    train_generator = TimeseriesGenerator(scaled_train,scaled_train,length=n_input, batch_size=1)

    model = Sequential()

    model.add(LSTM(128,activation = 'relu', input_shape= (n_input, n_feature), return_sequences=True))
    model.add(LSTM(128, activation='relu', return_sequences=True))
    model.add(LSTM(128, activation='relu', return_sequences=False))
    model.add(Dense(1))
    model.compile(optimizer='adam', loss='mse')
    history =model.fit_generator(train_generator,epochs=50, batch_size=1,verbose=1)

# Get a SageMaker-compatible role used by this Notebook Instance.
    role = get_execution_role()
    with open(output_model_dir + '/history.json', 'w') as f:
         json.dump(history.history, f)
    #Save the Scaler
    dump(scaler, output_model_dir + '/scaler.model', protocol=2) 
    #Save the trained model and weights
    model_json = model.to_json()
    with open(output_model_dir + "/model.json", "w") as json_file:
        json_file.write(model_json)
    model.save_weights(output_model_dir + "/model.h5")

这里显示一些错误：

 train_instance_type = "ml.m4.xlarg"

tf_estimator = TensorFlow(entry_point='time_series.py', role=get_execution_role(),
                          train_instance_count=1, train_instance_type=train_instance_type,
                          framework_version='1.12', py_version='py3', script_mode=True,
                          output_path = 's3://' + s3Bucket, base_job_name = "sales-forecasting-lstm",
                         hyperparameters={'batch_size': 2,
                                           'epochs': 50})

tf_estimator.fit(uploaded_data_path)

这里我得到了错误。这是什么错误，我不明白这个错误。

UnexpectedStatusException: Error for Training job sales-forecasting-lstm-2020-04-13-10-17-34-919: Failed. Reason: AlgorithmError: ExecuteUserScriptError:
Command "/usr/bin/python time_series.py --batch_size 2 --epochs 50 --model_dir s3://sagemaker12/sales-forecasting-lstm-2020-04-13-10-17-34-919/model"

​

嗨，我是 AWS sagemaker 的新手，我正在尝试在 sagemaker 中部署时间序列自定义 lstm 模型，请帮助我以及如何配置脚本模式，python 脚本进行部署。 这是我的脚本文件 timer_series.py 代码。

import sagemaker
import boto3
import os
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import tensorflow 
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
from tensorflow.keras.layers import LSTM
from sklearn.preprocessing import MinMaxScaler
from tensorflow.keras.preprocessing.sequence import TimeseriesGenerator

from sklearn.metrics import mean_squared_error


if __name__ =='__main__':

    parser.add_argument('--epochs', type=int, default=50)
    parser.add_argument('--batch_size', type=int, default=72)
    parser.add_argument('--n_train_hours', type=int, default=24*365*2)
    parser.add_argument('--n_validation_hours', type=int, default=24*365*4)

    # input data and model directories
    parser.add_argument('--model_dir', type=str)

    args, _ = parser.parse_known_args()

    train_dataset_dir = os.environ.get('SM_INPUT_DIR') + '/data/training/' 
    output_model_dir = os.environ.get('SM_MODEL_DIR')
    output_object_dir = os.environ.get('SM_OUTPUT_DATA_DIR')

    epochs = args.epochs
    batch_size = args.batch_size
    input_data = {args.input_data}
    dataset = read_csv( input_data + 'dataset.csv', header=0, index_col='Date')
    dataset.sort_index(inplace=True)
    train = dataset.iloc[:109]
    test= dataset.iloc[109:]  
    scaler = MinMaxScaler()
    scaled_train = scaler.fit_transform(train)
    scaled_test=scaler.fit_transform(test)
    n_input = 12
    n_feature = 1

    train_generator = TimeseriesGenerator(scaled_train,scaled_train,length=n_input, batch_size=1)

    model = Sequential()

    model.add(LSTM(128,activation = 'relu', input_shape= (n_input, n_feature), return_sequences=True))
    model.add(LSTM(128, activation='relu', return_sequences=True))
    model.add(LSTM(128, activation='relu', return_sequences=False))
    model.add(Dense(1))
    model.compile(optimizer='adam', loss='mse')
    history =model.fit_generator(train_generator,epochs=50, batch_size=1,verbose=1)

# Get a SageMaker-compatible role used by this Notebook Instance.
    role = get_execution_role()
    with open(output_model_dir + '/history.json', 'w') as f:
         json.dump(history.history, f)
    #Save the Scaler
    dump(scaler, output_model_dir + '/scaler.model', protocol=2) 
    #Save the trained model and weights
    model_json = model.to_json()
    with open(output_model_dir + "/model.json", "w") as json_file:
        json_file.write(model_json)
    model.save_weights(output_model_dir + "/model.h5")

这里显示一些错误：

 train_instance_type = "ml.m4.xlarg"

tf_estimator = TensorFlow(entry_point='time_series.py', role=get_execution_role(),
                          train_instance_count=1, train_instance_type=train_instance_type,
                          framework_version='1.12', py_version='py3', script_mode=True,
                          output_path = 's3://' + s3Bucket, base_job_name = "sales-forecasting-lstm",
                         hyperparameters={'batch_size': 2,
                                           'epochs': 50})

tf_estimator.fit(uploaded_data_path)

这里我得到了错误。这是什么错误，我不明白这个错误。

UnexpectedStatusException: Error for Training job sales-forecasting-lstm-2020-04-13-10-17-34-919: Failed. Reason: AlgorithmError: ExecuteUserScriptError:
Command "/usr/bin/python time_series.py --batch_size 2 --epochs 50 --model_dir s3://sagemaker12/sales-forecasting-lstm-2020-04-13-10-17-34-919/model"

​

Answer 1

我建议您将 timer_series.py 通道更改为您的 s3 路径，您不必在 train_dataset_dir 上添加加号并将 sagemaker 特定参数添加为：

parser.add_argument('--output-data-dir', type=str, default='s3://bucket_name/folder_name/output')

上面的行用于指示应将输出数据存储在何处。但是，这是您在调用批量转换函数时必须指定的内容。否则会将其存储在默认存储桶中。 其次，为了调试并能够帮助您，您必须查看 CloudWatch 的特定培训作业，以更好地了解您的脚本失败的原因：time_series.py。我想你的训练数据的规格和阅读有问题。