TabNet量化选股策略¶

策略思想¶

根据TabNet的预测结果，选择20支股票买入进行持有，每5日换仓。

In [ ]:

# 本代码由可视化策略环境自动生成 2022年8月18日 11:47
# 本代码单元只能在可视化模式下编辑。您也可以拷贝代码，粘贴到新建的代码单元或者策略，然后修改。


# Python 代码入口函数，input_1/2/3 对应三个输入端，data_1/2/3 对应三个输出端
def m10_run_bigquant_run(input_1, input_2, input_3):
    # 示例代码如下。在这里编写您的代码
    from sklearn.model_selection import train_test_split
    data = input_1.read()
    x_train, x_val, y_train, y_val = train_test_split(data["x"], data['y'], test_size=0.2)
    data_1 = DataSource.write_pickle({'x': x_train, 'y': y_train})
    data_2 = DataSource.write_pickle({'x': x_val, 'y': y_val})
    return Outputs(data_1=data_1, data_2=data_2, data_3=None)

# 后处理函数，可选。输入是主函数的输出，可以在这里对数据做处理，或者返回更友好的outputs数据格式。此函数输出不会被缓存。
def m10_post_run_bigquant_run(outputs):
    return outputs

import tensorflow as tf
from tensorflow.keras.layers import Layer

def glu(x, n_units=None):
    """Generalized linear unit nonlinear activation."""
    if n_units is None:
        n_units = tf.shape(x)[-1] // 2

    return x[..., :n_units] * tf.nn.sigmoid(x[..., n_units:])


def sparsemax(logits, axis):
    logits = tf.convert_to_tensor(logits, name="logits")

    # We need its original shape for shape inference.
    shape = logits.get_shape()
    rank = shape.rank
    is_last_axis = (axis == -1) or (axis == rank - 1)

    if is_last_axis:
        output = _compute_2d_sparsemax(logits)
        output.set_shape(shape)
        return output

    # Swap logits' dimension of dim and its last dimension.
    rank_op = tf.rank(logits)
    axis_norm = axis % rank
    logits = _swap_axis(logits, axis_norm, tf.math.subtract(rank_op, 1))

    # Do the actual softmax on its last dimension.
    output = _compute_2d_sparsemax(logits)
    output = _swap_axis(output, axis_norm, tf.math.subtract(rank_op, 1))

    # Make shape inference work since transpose may erase its static shape.
    output.set_shape(shape)
    return output


def _swap_axis(logits, dim_index, last_index, **kwargs):
    return tf.transpose(
        logits,
        tf.concat(
            [
                tf.range(dim_index),
                [last_index],
                tf.range(dim_index + 1, last_index),
                [dim_index],
            ],
            0,
        ),
        **kwargs,
    )


def _compute_2d_sparsemax(logits):
    """Performs the sparsemax operation when axis=-1."""
    shape_op = tf.shape(logits)
    obs = tf.math.reduce_prod(shape_op[:-1])
    dims = shape_op[-1]

    # In the paper, they call the logits z.
    # The mean(logits) can be substracted from logits to make the algorithm
    # more numerically stable. the instability in this algorithm comes mostly
    # from the z_cumsum. Substacting the mean will cause z_cumsum to be close
    # to zero. However, in practise the numerical instability issues are very
    # minor and substacting the mean causes extra issues with inf and nan
    # input.
    # Reshape to [obs, dims] as it is almost free and means the remanining
    # code doesn't need to worry about the rank.
    z = tf.reshape(logits, [obs, dims])

    # sort z
    z_sorted, _ = tf.nn.top_k(z, k=dims)

    # calculate k(z)
    z_cumsum = tf.math.cumsum(z_sorted, axis=-1)
    k = tf.range(1, tf.cast(dims, logits.dtype) + 1) #, dtype=logits.dtype)
    z_check = 1 + k * z_sorted > z_cumsum
    # because the z_check vector is always [1,1,...1,0,0,...0] finding the
    # (index + 1) of the last `1` is the same as just summing the number of 1.
    k_z = tf.math.reduce_sum(tf.cast(z_check, tf.int32), axis=-1)

    # calculate tau(z)
    # If there are inf values or all values are -inf, the k_z will be zero,
    # this is mathematically invalid and will also cause the gather_nd to fail.
    # Prevent this issue for now by setting k_z = 1 if k_z = 0, this is then
    # fixed later (see p_safe) by returning p = nan. This results in the same
    # behavior as softmax.
    k_z_safe = tf.math.maximum(k_z, 1)
    indices = tf.stack([tf.range(0, obs), tf.reshape(k_z_safe, [-1]) - 1], axis=1)
    tau_sum = tf.gather_nd(z_cumsum, indices)
    tau_z = (tau_sum - 1) / tf.cast(k_z, logits.dtype)

    # calculate p
    p = tf.math.maximum(tf.cast(0, logits.dtype), z - tf.expand_dims(tau_z, -1))
    # If k_z = 0 or if z = nan, then the input is invalid
    p_safe = tf.where(
        tf.expand_dims(
            tf.math.logical_or(tf.math.equal(k_z, 0), tf.math.is_nan(z_cumsum[:, -1])),
            axis=-1,
        ),
        tf.fill([obs, dims], tf.cast(float("nan"), logits.dtype)),
        p,
    )

    # Reshape back to original size
    p_safe = tf.reshape(p_safe, shape_op)
    return p_safe


"""
Code replicated from https://github.com/tensorflow/addons/blob/master/tensorflow_addons/layers/normalizations.py
"""
class GroupNormalization(tf.keras.layers.Layer):
    def __init__(
            self,
            groups: int = 2,
            axis: int = -1,
            epsilon: float = 1e-3,
            center: bool = True,
            scale: bool = True,
            beta_initializer="zeros",
            gamma_initializer="ones",
            beta_regularizer=None,
            gamma_regularizer=None,
            beta_constraint=None,
            gamma_constraint=None,
            **kwargs
    ):
        super().__init__(**kwargs)
        self.supports_masking = True
        self.groups = groups
        self.axis = axis
        self.epsilon = epsilon
        self.center = center
        self.scale = scale
        self.beta_initializer = tf.keras.initializers.get(beta_initializer)
        self.gamma_initializer = tf.keras.initializers.get(gamma_initializer)
        self.beta_regularizer = tf.keras.regularizers.get(beta_regularizer)
        self.gamma_regularizer = tf.keras.regularizers.get(gamma_regularizer)
        self.beta_constraint = tf.keras.constraints.get(beta_constraint)
        self.gamma_constraint = tf.keras.constraints.get(gamma_constraint)
        self._check_axis()

    def build(self, input_shape):

        self._check_if_input_shape_is_none(input_shape)
        self._set_number_of_groups_for_instance_norm(input_shape)
        self._check_size_of_dimensions(input_shape)
        self._create_input_spec(input_shape)

        self._add_gamma_weight(input_shape)
        self._add_beta_weight(input_shape)
        self.built = True
        super().build(input_shape)

    def call(self, inputs, training=None):
        # Training=none is just for compat with batchnorm signature call
        input_shape = tf.keras.backend.int_shape(inputs)
        tensor_input_shape = tf.shape(inputs)

        reshaped_inputs, group_shape = self._reshape_into_groups(
            inputs, input_shape, tensor_input_shape
        )

        normalized_inputs = self._apply_normalization(reshaped_inputs, input_shape)

        outputs = tf.reshape(normalized_inputs, tensor_input_shape)

        return outputs

    def get_config(self):
        config = {
            "groups": self.groups,
            "axis": self.axis,
            "epsilon": self.epsilon,
            "center": self.center,
            "scale": self.scale,
            "beta_initializer": tf.keras.initializers.serialize(self.beta_initializer),
            "gamma_initializer": tf.keras.initializers.serialize(
                self.gamma_initializer
            ),
            "beta_regularizer": tf.keras.regularizers.serialize(self.beta_regularizer),
            "gamma_regularizer": tf.keras.regularizers.serialize(
                self.gamma_regularizer
            ),
            "beta_constraint": tf.keras.constraints.serialize(self.beta_constraint),
            "gamma_constraint": tf.keras.constraints.serialize(self.gamma_constraint),
        }
        base_config = super().get_config()
        return {**base_config, **config}

    def compute_output_shape(self, input_shape):
        return input_shape

    def _reshape_into_groups(self, inputs, input_shape, tensor_input_shape):

        group_shape = [tensor_input_shape[i] for i in range(len(input_shape))]
        group_shape[self.axis] = input_shape[self.axis] // self.groups
        group_shape.insert(self.axis, self.groups)
        group_shape = tf.stack(group_shape)
        reshaped_inputs = tf.reshape(inputs, group_shape)
        return reshaped_inputs, group_shape

    def _apply_normalization(self, reshaped_inputs, input_shape):

        group_shape = tf.keras.backend.int_shape(reshaped_inputs)
        group_reduction_axes = list(range(1, len(group_shape)))
        axis = -2 if self.axis == -1 else self.axis - 1
        group_reduction_axes.pop(axis)

        mean, variance = tf.nn.moments(
            reshaped_inputs, group_reduction_axes, keepdims=True
        )

        gamma, beta = self._get_reshaped_weights(input_shape)
        normalized_inputs = tf.nn.batch_normalization(
            reshaped_inputs,
            mean=mean,
            variance=variance,
            scale=gamma,
            offset=beta,
            variance_epsilon=self.epsilon,
        )
        return normalized_inputs

    def _get_reshaped_weights(self, input_shape):
        broadcast_shape = self._create_broadcast_shape(input_shape)
        gamma = None
        beta = None
        if self.scale:
            gamma = tf.reshape(self.gamma, broadcast_shape)

        if self.center:
            beta = tf.reshape(self.beta, broadcast_shape)
        return gamma, beta

    def _check_if_input_shape_is_none(self, input_shape):
        dim = input_shape[self.axis]
        if dim is None:
            raise ValueError(
                "Axis " + str(self.axis) + " of "
                                           "input tensor should have a defined dimension "
                                           "but the layer received an input with shape " + str(input_shape) + "."
            )

    def _set_number_of_groups_for_instance_norm(self, input_shape):
        dim = input_shape[self.axis]

        if self.groups == -1:
            self.groups = dim

    def _check_size_of_dimensions(self, input_shape):

        dim = input_shape[self.axis]
        if dim < self.groups:
            raise ValueError(
                "Number of groups (" + str(self.groups) + ") cannot be "
                                                          "more than the number of channels (" + str(dim) + ")."
            )

        if dim % self.groups != 0:
            raise ValueError(
                "Number of groups (" + str(self.groups) + ") must be a "
                                                          "multiple of the number of channels (" + str(dim) + ")."
            )

    def _check_axis(self):

        if self.axis == 0:
            raise ValueError(
                "You are trying to normalize your batch axis. Do you want to "
                "use tf.layer.batch_normalization instead"
            )

    def _create_input_spec(self, input_shape):

        dim = input_shape[self.axis]
        self.input_spec = tf.keras.layers.InputSpec(
            ndim=len(input_shape), axes={self.axis: dim}
        )

    def _add_gamma_weight(self, input_shape):

        dim = input_shape[self.axis]
        shape = (dim,)

        if self.scale:
            self.gamma = self.add_weight(
                shape=shape,
                name="gamma",
                initializer=self.gamma_initializer,
                regularizer=self.gamma_regularizer,
                constraint=self.gamma_constraint,
            )
        else:
            self.gamma = None

    def _add_beta_weight(self, input_shape):

        dim = input_shape[self.axis]
        shape = (dim,)

        if self.center:
            self.beta = self.add_weight(
                shape=shape,
                name="beta",
                initializer=self.beta_initializer,
                regularizer=self.beta_regularizer,
                constraint=self.beta_constraint,
            )
        else:
            self.beta = None

    def _create_broadcast_shape(self, input_shape):
        broadcast_shape = [1] * len(input_shape)
        broadcast_shape[self.axis] = input_shape[self.axis] // self.groups
        broadcast_shape.insert(self.axis, self.groups)
        return broadcast_shape

    
class TransformBlock(tf.keras.layers.Layer):

    def __init__(self, features,
                 norm_type,
                 momentum=0.9,
                 virtual_batch_size=None,
                 groups=2,
                 block_name='',
                 **kwargs):
        super(TransformBlock, self).__init__(**kwargs)

        self.features = features
        self.norm_type = norm_type
        self.momentum = momentum
        self.groups = groups
        self.virtual_batch_size = virtual_batch_size
        self.block_name = block_name
    
    def build(self, input_shape):
        self.transform = tf.keras.layers.Dense(self.features, use_bias=False, name=f'transformblock_dense_{self.block_name}')
        if self.norm_type == 'batch':
            self.bn = tf.keras.layers.BatchNormalization(axis=-1, momentum=momentum,
                                                         virtual_batch_size=virtual_batch_size,
                                                         name=f'transformblock_bn_{self.block_name}')
        else:
            self.bn = GroupNormalization(axis=-1, groups=self.groups, name=f'transformblock_gn_{self.block_name}')
            
        self.built = True
        super().build(input_shape)
        
    def call(self, inputs, training=None):
        x = self.transform(inputs)
        x = self.bn(x, training=training)
        return x
    
    def get_config(self):
        config = {
            "features": self.features,
            "norm_type": self.norm_type,
            "virtual_batch_size": self.virtual_batch_size,
            "groups": self.groups,
            "block_name": self.block_name
        }
        base_config = super().get_config()
        return {**base_config, **config}
    
    def compute_output_shape(self, input_shape):
        return input_shape


class TabNetEncoderLayer(tf.keras.layers.Layer):

    def __init__(self, feature_columns,
                 feature_dim=16,
                 output_dim=8,
                 num_features=None,
                 num_decision_steps=3,
                 relaxation_factor=1.5,
                 sparsity_coefficient=1e-5,
                 norm_type='group',
                 batch_momentum=0.98,
                 virtual_batch_size=1024,
                 num_groups=2,
                 epsilon=1e-5,
                 **kwargs):

        super(TabNetEncoderLayer, self).__init__(**kwargs)

        # Input checks
        if feature_columns is not None:
            if type(feature_columns) not in (list, tuple):
                raise ValueError("`feature_columns` must be a list or a tuple.")

            if len(feature_columns) == 0:
                raise ValueError("`feature_columns` must be contain at least 1 tf.feature_column !")

            if num_features is None:
                num_features = len(feature_columns)
            else:
                num_features = int(num_features)

        else:
            if num_features is None:
                raise ValueError("If `feature_columns` is None, then `num_features` cannot be None.")

        if num_decision_steps < 1:
            raise ValueError("Num decision steps must be greater than 0.")
        
        if feature_dim <= output_dim:
            raise ValueError("To compute `features_for_coef`, feature_dim must be larger than output dim")

        feature_dim = int(feature_dim)
        output_dim = int(output_dim)
        num_decision_steps = int(num_decision_steps)
        relaxation_factor = float(relaxation_factor)
        sparsity_coefficient = float(sparsity_coefficient)
        batch_momentum = float(batch_momentum)
        num_groups = max(1, int(num_groups))
        epsilon = float(epsilon)

        if relaxation_factor < 0.:
            raise ValueError("`relaxation_factor` cannot be negative !")

        if sparsity_coefficient < 0.:
            raise ValueError("`sparsity_coefficient` cannot be negative !")

        if virtual_batch_size is not None:
            virtual_batch_size = int(virtual_batch_size)

        if norm_type not in ['batch', 'group']:
            raise ValueError("`norm_type` must be either `batch` or `group`")

        self.feature_columns = feature_columns
        self.num_features = num_features
        self.feature_dim = feature_dim
        self.output_dim = output_dim

        self.num_decision_steps = num_decision_steps
        self.relaxation_factor = relaxation_factor
        self.sparsity_coefficient = sparsity_coefficient
        self.norm_type = norm_type
        self.batch_momentum = batch_momentum
        self.virtual_batch_size = virtual_batch_size
        self.num_groups = num_groups
        self.epsilon = epsilon

        if num_decision_steps > 1:
            features_for_coeff = feature_dim - output_dim
            print(f"[TabNet]: {features_for_coeff} features will be used for decision steps.")

        if self.feature_columns is not None:
            self.input_features = tf.keras.layers.DenseFeatures(feature_columns, trainable=True)

            if self.norm_type == 'batch':
                self.input_bn = tf.keras.layers.BatchNormalization(axis=-1, momentum=batch_momentum, name='input_bn')
            else:
                self.input_bn = GroupNormalization(axis=-1, groups=self.num_groups, name='input_gn')

        else:
            self.input_features = None
            self.input_bn = None
    
    def build(self, input_shape):
        self.transform_f1 = TransformBlock(2 * self.feature_dim, self.norm_type,
                                           self.batch_momentum, self.virtual_batch_size, self.num_groups,
                                           block_name='f1')

        self.transform_f2 = TransformBlock(2 * self.feature_dim, self.norm_type,
                                           self.batch_momentum, self.virtual_batch_size, self.num_groups,
                                           block_name='f2')

        self.transform_f3_list = [
            TransformBlock(2 * self.feature_dim, self.norm_type,
                           self.batch_momentum, self.virtual_batch_size, self.num_groups, block_name=f'f3_{i}')
            for i in range(self.num_decision_steps)
        ]

        self.transform_f4_list = [
            TransformBlock(2 * self.feature_dim, self.norm_type,
                           self.batch_momentum, self.virtual_batch_size, self.num_groups, block_name=f'f4_{i}')
            for i in range(self.num_decision_steps)
        ]

        self.transform_coef_list = [
            TransformBlock(self.num_features, self.norm_type,
                           self.batch_momentum, self.virtual_batch_size, self.num_groups, block_name=f'coef_{i}')
            for i in range(self.num_decision_steps - 1)
        ]

        self._step_feature_selection_masks = None
        self._step_aggregate_feature_selection_mask = None
        self.built = True
        super(TabNetEncoderLayer, self).build(input_shape)

    def call(self, inputs, training=None):
        if self.input_features is not None:
            features = self.input_features(inputs)
            features = self.input_bn(features, training=training)

        else:
            features = inputs

        batch_size = tf.shape(features)[0]
        self._step_feature_selection_masks = []
        self._step_aggregate_feature_selection_mask = None

        # Initializes decision-step dependent variables.
        output_aggregated = tf.zeros([batch_size, self.output_dim])
        masked_features = features
        mask_values = tf.zeros([batch_size, self.num_features])
        aggregated_mask_values = tf.zeros([batch_size, self.num_features])
        complementary_aggregated_mask_values = tf.ones(
            [batch_size, self.num_features])

        total_entropy = 0.0
        entropy_loss = 0.

        for ni in range(self.num_decision_steps):
            # Feature transformer with two shared and two decision step dependent
            # blocks is used below.=
            transform_f1 = self.transform_f1(masked_features, training=training)
            transform_f1 = glu(transform_f1, self.feature_dim)

            transform_f2 = self.transform_f2(transform_f1, training=training)
            transform_f2 = (glu(transform_f2, self.feature_dim) +
                            transform_f1) * tf.math.sqrt(0.5)

            transform_f3 = self.transform_f3_list[ni](transform_f2, training=training)
            transform_f3 = (glu(transform_f3, self.feature_dim) +
                            transform_f2) * tf.math.sqrt(0.5)

            transform_f4 = self.transform_f4_list[ni](transform_f3, training=training)
            transform_f4 = (glu(transform_f4, self.feature_dim) +
                            transform_f3) * tf.math.sqrt(0.5)

            if (ni > 0 or self.num_decision_steps == 1):
                decision_out = tf.nn.relu(transform_f4[:, :self.output_dim])

                # Decision aggregation.
                output_aggregated += decision_out

                # Aggregated masks are used for visualization of the
                # feature importance attributes.
                scale_agg = tf.reduce_sum(decision_out, axis=1, keepdims=True)

                if self.num_decision_steps > 1:
                    scale_agg = scale_agg / tf.cast(self.num_decision_steps - 1, tf.float32)

                aggregated_mask_values += mask_values * scale_agg

            features_for_coef = transform_f4[:, self.output_dim:]

            if ni < (self.num_decision_steps - 1):
                # Determines the feature masks via linear and nonlinear
                # transformations, taking into account of aggregated feature use.
                mask_values = self.transform_coef_list[ni](features_for_coef, training=training)
                mask_values *= complementary_aggregated_mask_values
                mask_values = sparsemax(mask_values, axis=-1)

                # Relaxation factor controls the amount of reuse of features between
                # different decision blocks and updated with the values of
                # coefficients.
                complementary_aggregated_mask_values *= (
                        self.relaxation_factor - mask_values)

                # Entropy is used to penalize the amount of sparsity in feature
                # selection.
                total_entropy += tf.reduce_mean(
                    tf.reduce_sum(
                        -mask_values * tf.math.log(mask_values + self.epsilon), axis=1)) / (
                                     tf.cast(self.num_decision_steps - 1, tf.float32))

                # Add entropy loss
                entropy_loss = total_entropy

                # Feature selection.
                masked_features = tf.multiply(mask_values, features)

                # Visualization of the feature selection mask at decision step ni
                # tf.summary.image(
                #     "Mask for step" + str(ni),
                #     tf.expand_dims(tf.expand_dims(mask_values, 0), 3),
                #     max_outputs=1)
                mask_at_step_i = tf.expand_dims(tf.expand_dims(mask_values, 0), 3)
                self._step_feature_selection_masks.append(mask_at_step_i)

            else:
                # This branch is needed for correct compilation by tf.autograph
                entropy_loss = 0.

        # Adds the loss automatically
        self.add_loss(self.sparsity_coefficient * entropy_loss)

        # Visualization of the aggregated feature importances
        # tf.summary.image(
        #     "Aggregated mask",
        #     tf.expand_dims(tf.expand_dims(aggregated_mask_values, 0), 3),
        #     max_outputs=1)

        agg_mask = tf.expand_dims(tf.expand_dims(aggregated_mask_values, 0), 3)
        self._step_aggregate_feature_selection_mask = agg_mask
        return output_aggregated

    def feature_selection_masks(self):
        return self._step_feature_selection_masks

    def aggregate_feature_selection_mask(self):
        return self._step_aggregate_feature_selection_mask
    
    def compute_output_shape(self, input_shape):
        return self.output_dim
    
    def get_config(self):
        config = {
            "feature_columns": self.feature_columns,
            "num_features": self.num_features,
            "feature_dim": self.feature_dim,
            "output_dim": self.output_dim,
            "num_decision_steps": self.num_decision_steps,
            "relaxation_factor": self.relaxation_factor,
            "sparsity_coefficient": self.sparsity_coefficient,
            "norm_type": self.norm_type,
            "batch_momentum": self.batch_momentum,
            "virtual_batch_size": self.virtual_batch_size,
            "num_groups": self.num_groups,
            "epsilon": self.epsilon,
        }
        base_config = super().get_config()
        return {**base_config, **config}
    
    
# 必须也将 UserLayer 赋值给 m12_layer_class_bigquant_run
m12_layer_class_bigquant_run = TabNetEncoderLayer

from tensorflow.keras.optimizers import Adam, schedules

lr = schedules.ExponentialDecay(0.02, decay_steps=2000, decay_rate=0.9, staircase=False)

m5_user_optimizer_bigquant_run=Adam(lr)
from tensorflow.keras.callbacks import EarlyStopping

m5_earlystop_bigquant_run=EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=5)
# 用户的自定义层需要写到字典中，比如
# {
#   "MyLayer": MyLayer
# }
m5_custom_objects_bigquant_run = {
    "GroupNormalization": GroupNormalization,
    "TransformBlock": TransformBlock,
    "TabNetEncoderLayer": TabNetEncoderLayer
}

# Python 代码入口函数，input_1/2/3 对应三个输入端，data_1/2/3 对应三个输出端
def m24_run_bigquant_run(input_1, input_2, input_3):
    # 示例代码如下。在这里编写您的代码
    pred_label = input_1.read_pickle()
    df = input_2.read_df()
    df = pd.DataFrame({'pred_label':pred_label[:,0], 'instrument':df.instrument, 'date':df.date})
    df.sort_values(['date','pred_label'],inplace=True, ascending=[True,False])
    return Outputs(data_1=DataSource.write_df(df), data_2=None, data_3=None)

# 后处理函数，可选。输入是主函数的输出，可以在这里对数据做处理，或者返回更友好的outputs数据格式。此函数输出不会被缓存。
def m24_post_run_bigquant_run(outputs):
    return outputs

# 回测引擎：初始化函数，只执行一次
def m19_initialize_bigquant_run(context):
    # 加载预测数据
    context.ranker_prediction = context.options['data'].read_df()

    # 系统已经设置了默认的交易手续费和滑点，要修改手续费可使用如下函数
    context.set_commission(PerOrder(buy_cost=0.001, sell_cost=0.001, min_cost=5))
    # 预测数据，通过options传入进来，使用 read_df 函数，加载到内存 (DataFrame)
    # 设置买入的股票数量，这里买入预测股票列表排名靠前的5只
    stock_count = 2
    # 每只的股票的权重，如下的权重分配会使得靠前的股票分配多一点的资金，[0.339160, 0.213986, 0.169580, ..]
    context.stock_weights = T.norm([1 / math.log(i + 2) for i in range(0, stock_count)])
    # 设置每只股票占用的最大资金比例
    context.max_cash_per_instrument = 0.2
    context.options['hold_days'] = 5

# 回测引擎：每日数据处理函数，每天执行一次
def m19_handle_data_bigquant_run(context, data):
    # 按日期过滤得到今日的预测数据
    ranker_prediction = context.ranker_prediction[
        context.ranker_prediction.date == data.current_dt.strftime('%Y-%m-%d')]

    # 1. 资金分配
    # 平均持仓时间是hold_days，每日都将买入股票，每日预期使用 1/hold_days 的资金
    # 实际操作中，会存在一定的买入误差，所以在前hold_days天，等量使用资金；之后，尽量使用剩余资金（这里设置最多用等量的1.5倍）
    is_staging = context.trading_day_index < context.options['hold_days'] # 是否在建仓期间（前 hold_days 天）
    cash_avg = context.portfolio.portfolio_value / context.options['hold_days']
    cash_for_buy = min(context.portfolio.cash, (1 if is_staging else 1.5) * cash_avg)
    cash_for_sell = cash_avg - (context.portfolio.cash - cash_for_buy)
    positions = {e.symbol: p.amount * p.last_sale_price
                 for e, p in context.perf_tracker.position_tracker.positions.items()}

    # 2. 生成卖出订单：hold_days天之后才开始卖出；对持仓的股票，按机器学习算法预测的排序末位淘汰
    if not is_staging and cash_for_sell > 0:
        equities = {e.symbol: e for e, p in context.perf_tracker.position_tracker.positions.items()}
        instruments = list(reversed(list(ranker_prediction.instrument[ranker_prediction.instrument.apply(
                lambda x: x in equities and not context.has_unfinished_sell_order(equities[x]))])))
        # print('rank order for sell %s' % instruments)
        for instrument in instruments:
            context.order_target(context.symbol(instrument), 0)
            cash_for_sell -= positions[instrument]
            if cash_for_sell <= 0:
                break

    # 3. 生成买入订单：按机器学习算法预测的排序，买入前面的stock_count只股票
    buy_cash_weights = context.stock_weights
    buy_instruments = list(ranker_prediction.instrument[:len(buy_cash_weights)])
    max_cash_per_instrument = context.portfolio.portfolio_value * context.max_cash_per_instrument
    for i, instrument in enumerate(buy_instruments):
        cash = cash_for_buy * buy_cash_weights[i]
        if cash > max_cash_per_instrument - positions.get(instrument, 0):
            # 确保股票持仓量不会超过每次股票最大的占用资金量
            cash = max_cash_per_instrument - positions.get(instrument, 0)
        if cash > 0:
            context.order_value(context.symbol(instrument), cash)

# 回测引擎：准备数据，只执行一次
def m19_prepare_bigquant_run(context):
    pass


m1 = M.instruments.v2(
    start_date='2014-01-01',
    end_date='2018-12-31',
    market='CN_STOCK_A',
    instrument_list='',
    max_count=0
)

m2 = M.advanced_auto_labeler.v2(
    instruments=m1.data,
    label_expr="""# #号开始的表示注释
# 0. 每行一个，顺序执行，从第二个开始，可以使用label字段
# 1. 可用数据字段见 https://bigquant.com/docs/data_history_data.html
#   添加benchmark_前缀，可使用对应的benchmark数据
# 2. 可用操作符和函数见 `表达式引擎 <https://bigquant.com/docs/big_expr.html>`_

# 计算收益：5日收盘价(作为卖出价格)除以明日开盘价(作为买入价格)
shift(close, -5) / shift(open, -1)-1

# 极值处理：用1%和99%分位的值做clip
clip(label, all_quantile(label, 0.01), all_quantile(label, 0.99))

# 过滤掉一字涨停的情况 (设置label为NaN，在后续处理和训练中会忽略NaN的label)
where(shift(high, -1) == shift(low, -1), NaN, label)
""",
    start_date='',
    end_date='',
    benchmark='000300.SHA',
    drop_na_label=True,
    cast_label_int=False
)

m29 = M.standardlize.v8(
    input_1=m2.data,
    columns_input='label'
)

m3 = M.input_features.v1(
    features="""
close_0
open_0
high_0
low_0 
amount_0
turn_0 
return_0
 
close_1
open_1
high_1
low_1
return_1
amount_1
turn_1
 
close_2
open_2
high_2
low_2
amount_2
turn_2
return_2
 
close_3
open_3
high_3
low_3
amount_3
turn_3
return_3
 
close_4
open_4
high_4
low_4
amount_4
turn_4
return_4
 

mean(close_0,   5)
mean(low_0,   5)
mean(open_0,   5)
mean(high_0,   5)
mean(turn_0,   5)
mean(amount_0,   5)
mean(return_0,   5)
 
ts_max(close_0,   5)
ts_max(low_0,   5)
ts_max(open_0,   5)
ts_max(high_0,   5)
ts_max(turn_0,   5)
ts_max(amount_0,   5)
ts_max(return_0,   5)
 
ts_min(close_0,   5)
ts_min(low_0,   5)
ts_min(open_0,   5)
ts_min(high_0,   5)
ts_min(turn_0,   5)
ts_min(amount_0,   5)
ts_min(return_0,   5) 
 
std(close_0,   5)
std(low_0,   5)
std(open_0,   5)
std(high_0,   5)
std(turn_0,   5)
std(amount_0,   5)
std(return_0,   5)
 
ts_rank(close_0,   5)
ts_rank(low_0,   5)
ts_rank(open_0,   5)
ts_rank(high_0,   5)
ts_rank(turn_0,   5)
ts_rank(amount_0,   5)
ts_rank(return_0,   5)
 
decay_linear(close_0,   5)
decay_linear(low_0,   5)
decay_linear(open_0,   5)
decay_linear(high_0,   5)
decay_linear(turn_0,   5)
decay_linear(amount_0,   5)
decay_linear(return_0,   5)
 
correlation(volume_0, return_0,   5)
correlation(volume_0, high_0,   5)
correlation(volume_0, low_0,   5)
correlation(volume_0, close_0,   5)
correlation(volume_0, open_0,   5)
correlation(volume_0, turn_0,   5)
  
correlation(return_0, high_0,   5)
correlation(return_0, low_0,   5)
correlation(return_0, close_0,   5)
correlation(return_0, open_0,   5)
correlation(return_0, turn_0,   5)
 
correlation(high_0, low_0,   5)
correlation(high_0, close_0,   5)
correlation(high_0, open_0,   5)
correlation(high_0, turn_0,   5)
 
correlation(low_0, close_0,   5)
correlation(low_0, open_0,   5)
correlation(low_0, turn_0,   5)
 
correlation(close_0, open_0,   5)
correlation(close_0, turn_0,   5)

correlation(open_0, turn_0,   5)
"""
)

m15 = M.general_feature_extractor.v7(
    instruments=m1.data,
    features=m3.data,
    start_date='',
    end_date='',
    before_start_days=0
)

m16 = M.derived_feature_extractor.v3(
    input_data=m15.data,
    features=m3.data,
    date_col='date',
    instrument_col='instrument',
    drop_na=True,
    remove_extra_columns=False
)

m28 = M.standardlize.v8(
    input_1=m16.data,
    input_2=m3.data,
    columns_input='[]'
)

m13 = M.fillnan.v1(
    input_data=m28.data,
    features=m3.data,
    fill_value='0.0'
)

m7 = M.join.v3(
    data1=m29.data,
    data2=m13.data,
    on='date,instrument',
    how='inner',
    sort=False
)

m26 = M.dl_convert_to_bin.v2(
    input_data=m7.data,
    features=m3.data,
    window_size=1,
    feature_clip=3,
    flatten=True,
    window_along_col='instrument'
)

m10 = M.cached.v3(
    input_1=m26.data,
    run=m10_run_bigquant_run,
    post_run=m10_post_run_bigquant_run,
    input_ports='',
    params='{}',
    output_ports=''
)

m9 = M.instruments.v2(
    start_date=T.live_run_param('trading_date', '2019-01-01'),
    end_date=T.live_run_param('trading_date', '2022-08-19'),
    market='CN_STOCK_A',
    instrument_list="""002719.SZA
600063.SHA
002903.SZA
600753.SHA
600751.SHA
600750.SHA
600749.SHA
600748.SHA
600746.SHA
000001.SZA
600745.SHA
600744.SHA
600743.SHA
600742.SHA
600741.SHA
600740.SHA
600754.SHA
600755.SHA
600756.SHA
600757.SHA




002405.SZA
603126.SHA
002364.SZA
002363.SZA
000045.SZA
002221.SZA
600889.SHA
002662.SZA
000912.SZA
300421.SZA
003021.SZA
002345.SZA
605288.SHA
002965.SZA
002768.SZA
301021.SZA
600673.SHA
000049.SZA
300856.SZA
603212.SHA

""",
    max_count=0
)

m17 = M.general_feature_extractor.v7(
    instruments=m9.data,
    features=m3.data,
    start_date='',
    end_date='',
    before_start_days=0
)

m18 = M.derived_feature_extractor.v3(
    input_data=m17.data,
    features=m3.data,
    date_col='date',
    instrument_col='instrument',
    drop_na=True,
    remove_extra_columns=False
)

m25 = M.standardlize.v8(
    input_1=m18.data,
    input_2=m3.data,
    columns_input='[]'
)

m14 = M.fillnan.v1(
    input_data=m25.data,
    features=m3.data,
    fill_value='0.0'
)

m27 = M.dl_convert_to_bin.v2(
    input_data=m14.data,
    features=m3.data,
    window_size=1,
    feature_clip=3,
    flatten=True,
    window_along_col='instrument'
)

m6 = M.dl_layer_input.v1(
    shape='98',
    batch_shape='',
    dtype='float32',
    sparse=False,
    name=''
)

m12 = M.dl_layer_userlayer.v1(
    input1=m6.data,
    layer_class=m12_layer_class_bigquant_run,
    params="""{
    "num_features": 98, 
    "feature_columns": None,
    "feature_dim": 64,
    "output_dim": 32,
    "num_decision_steps": 3,
    "relaxation_factor": 1.3,
    "sparsity_coefficient": 1e-5,
    "norm_type": "group",
    "batch_momentum": 0.9,
    "virtual_batch_size": 128,
    "num_groups": 2,
    "epsilon": 1e-5
}""",
    name=''
)

m23 = M.dl_layer_dense.v1(
    inputs=m12.data,
    units=1,
    activation='linear',
    use_bias=False,
    kernel_initializer='Zeros',
    bias_initializer='Zeros',
    kernel_regularizer='None',
    kernel_regularizer_l1=0,
    kernel_regularizer_l2=0,
    bias_regularizer='None',
    bias_regularizer_l1=0,
    bias_regularizer_l2=0,
    activity_regularizer='None',
    activity_regularizer_l1=0,
    activity_regularizer_l2=0,
    kernel_constraint='None',
    bias_constraint='None',
    name=''
)

m4 = M.dl_model_init.v1(
    inputs=m6.data,
    outputs=m23.data
)

m5 = M.dl_model_train.v1(
    input_model=m4.data,
    training_data=m10.data_1,
    validation_data=m10.data_2,
    optimizer='自定义',
    user_optimizer=m5_user_optimizer_bigquant_run,
    loss='mean_squared_error',
    metrics='mse',
    batch_size=10240,
    epochs=100,
    earlystop=m5_earlystop_bigquant_run,
    custom_objects=m5_custom_objects_bigquant_run,
    n_gpus=0,
    verbose='2:每个epoch输出一行记录',
    m_cached=False
)

m11 = M.dl_model_predict.v1(
    trained_model=m5.data,
    input_data=m27.data,
    batch_size=1024,
    n_gpus=0,
    verbose='2:每个epoch输出一行记录'
)

m24 = M.cached.v3(
    input_1=m11.data,
    input_2=m18.data,
    run=m24_run_bigquant_run,
    post_run=m24_post_run_bigquant_run,
    input_ports='',
    params='{}',
    output_ports=''
)

m19 = M.trade.v4(
    instruments=m9.data,
    options_data=m24.data_1,
    start_date='',
    end_date='',
    initialize=m19_initialize_bigquant_run,
    handle_data=m19_handle_data_bigquant_run,
    prepare=m19_prepare_bigquant_run,
    volume_limit=0.025,
    order_price_field_buy='open',
    order_price_field_sell='close',
    capital_base=1000000,
    auto_cancel_non_tradable_orders=True,
    data_frequency='daily',
    price_type='真实价格',
    product_type='股票',
    plot_charts=True,
    backtest_only=False,
    benchmark='000300.SHA'
)

[2022-08-18 11:37:12.219988] INFO: moduleinvoker: instruments.v2 开始运行..

[2022-08-18 11:37:12.253425] INFO: moduleinvoker: 命中缓存

[2022-08-18 11:37:12.256007] INFO: moduleinvoker: instruments.v2 运行完成[0.036029s].

[2022-08-18 11:37:12.320197] INFO: moduleinvoker: advanced_auto_labeler.v2 开始运行..

[2022-08-18 11:37:12.326874] INFO: moduleinvoker: 命中缓存

[2022-08-18 11:37:12.860108] INFO: moduleinvoker: advanced_auto_labeler.v2 运行完成[0.539907s].

[2022-08-18 11:37:12.886000] INFO: moduleinvoker: standardlize.v8 开始运行..

[2022-08-18 11:37:12.893930] INFO: moduleinvoker: 命中缓存

[2022-08-18 11:37:12.897342] INFO: moduleinvoker: standardlize.v8 运行完成[0.011341s].

[2022-08-18 11:37:12.912054] INFO: moduleinvoker: input_features.v1 开始运行..

[2022-08-18 11:37:12.919502] INFO: moduleinvoker: 命中缓存

[2022-08-18 11:37:12.922560] INFO: moduleinvoker: input_features.v1 运行完成[0.010444s].

[2022-08-18 11:37:12.949643] INFO: moduleinvoker: general_feature_extractor.v7 开始运行..

[2022-08-18 11:37:12.957289] INFO: moduleinvoker: 命中缓存

[2022-08-18 11:37:12.960018] INFO: moduleinvoker: general_feature_extractor.v7 运行完成[0.010396s].

[2022-08-18 11:37:12.988751] INFO: moduleinvoker: derived_feature_extractor.v3 开始运行..

[2022-08-18 11:37:12.995965] INFO: moduleinvoker: 命中缓存

[2022-08-18 11:37:12.999176] INFO: moduleinvoker: derived_feature_extractor.v3 运行完成[0.010419s].

[2022-08-18 11:37:13.008332] INFO: moduleinvoker: standardlize.v8 开始运行..