utils.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Jul  7 23:08:26 2018

@author: bmj
"""

import gc
import time
from time import strftime,gmtime
import numpy as np
import pandas as pd
import os
load = False
cache_path = './cache3/'

from time import strftime,gmtime


def concat(L):
    result = None
    for l in L:
        if result is None:
            result = l
        else:
            try:
                result[l.columns.tolist()] = l
            except:
                print(l.head())
    return result


def left_merge(data1,data2,on):
    if type(on) != list:
        on = [on]
    if (set(on) & set(data2.columns)) != set(on):
        data2_temp = data2.reset_index()
    else:
        data2_temp = data2.copy()
    columns = [f for f in data2.columns if f not in on]
    result = data1.merge(data2_temp,on=on,how='left')
    result = result[columns]
    return result


def get_feat_size(train,size_feat):
    """计算A组的数量大小（忽略NaN等价于count）"""
    result_path = cache_path +  ('_').join(size_feat)+'_feat_count'+'.hdf'
    if os.path.exists(result_path) & load:
        result = pd.read_hdf(result_path)
    else:
        result = train[size_feat].groupby(by=size_feat).size().reset_index().rename(columns={0: ('_').join(size_feat)+'_count'})
        result = left_merge(train,result,on=size_feat)
    return result


def get_feat_size_feat(train,base_feat,other_feat):
    """计算唯一计数（等价于unique count）"""
    result_path = cache_path + ('_').join(base_feat)+'_count_'+('_').join(other_feat)+'.hdf'
    if os.path.exists(result_path) & load:
        result = pd.read_hdf(result_path)
    else:
        result = train[base_feat].groupby(base_feat).size().reset_index()\
                      .groupby(other_feat).size().reset_index().rename(columns={0: ('_').join(base_feat)+'_count_'+('_').join(other_feat)})
        result = left_merge(train,result,on=other_feat)
    return result


def get_feat_stat_feat(train,base_feat,other_feat,stat_list=['min','max','var','size','mean','skew']):
    name = ('_').join(base_feat) + '_' + ('_').join(other_feat) + '_' + ('_').join(stat_list)
    result_path = cache_path + name +'.hdf'
    if os.path.exists(result_path) & load:
        result = pd.read_hdf(result_path)
    else:
        agg_dict = {}
        for stat in stat_list:
            agg_dict[name+stat] = stat
        result = train[base_feat + other_feat].groupby(base_feat)[",".join(other_feat)]\
        .agg(agg_dict)
        result = left_merge(train,result,on=base_feat)
    return result

def get_feat_ngroup(train,base_feat):
    name = ('_').join(base_feat)+'_ngroup'
    result_path = cache_path + ('_').join(base_feat)+'_ngroup'+'.hdf'
    if os.path.exists(result_path) & load:
        result = pd.read_hdf(result_path, 'w')
    else:
        train[name] = train.groupby(base_feat).ngroup()
        result = train[[name]]
        train.drop([name],axis=1,inplace=True)        
    return result