Upload 8 files

finnlp/utils/config.py

@@ -0,0 +1,77 @@
+# dir
+DATA_SAVE_DIR = "datasets"
+TRAINED_MODEL_DIR = "trained_models"
+TENSORBOARD_LOG_DIR = "tensorboard_log"
+RESULTS_DIR = "results"
+
+# date format: '%Y-%m-%d'
+TRAIN_START_DATE = "2014-01-01"
+TRAIN_END_DATE = "2020-07-31"
+
+TEST_START_DATE = "2020-08-01"
+TEST_END_DATE = "2021-10-01"
+
+TRADE_START_DATE = "2021-11-01"
+TRADE_END_DATE = "2021-12-01"
+
+# stockstats technical indicator column names
+# check https://pypi.org/project/stockstats/ for different names
+INDICATORS = [
+    "macd",
+    "boll_ub",
+    "boll_lb",
+    "rsi_30",
+    "cci_30",
+    "dx_30",
+    "close_30_sma",
+    "close_60_sma",
+]
+
+
+# Model Parameters
+A2C_PARAMS = {"n_steps": 5, "ent_coef": 0.01, "learning_rate": 0.0007}
+PPO_PARAMS = {
+    "n_steps": 2048,
+    "ent_coef": 0.01,
+    "learning_rate": 0.00025,
+    "batch_size": 64,
+}
+DDPG_PARAMS = {"batch_size": 128, "buffer_size": 50000, "learning_rate": 0.001}
+TD3_PARAMS = {
+    "batch_size": 100,
+    "buffer_size": 1000000,
+    "learning_rate": 0.001,
+}
+SAC_PARAMS = {
+    "batch_size": 64,
+    "buffer_size": 100000,
+    "learning_rate": 0.0001,
+    "learning_starts": 100,
+    "ent_coef": "auto_0.1",
+}
+ERL_PARAMS = {
+    "learning_rate": 3e-5,
+    "batch_size": 2048,
+    "gamma": 0.985,
+    "seed": 312,
+    "net_dimension": 512,
+    "target_step": 5000,
+    "eval_gap": 30,
+}
+RLlib_PARAMS = {"lr": 5e-5, "train_batch_size": 500, "gamma": 0.99}
+
+
+# Possible time zones
+TIME_ZONE_SHANGHAI = "Asia/Shanghai"  # Hang Seng HSI, SSE, CSI
+TIME_ZONE_USEASTERN = "US/Eastern"  # Dow, Nasdaq, SP
+TIME_ZONE_PARIS = "Europe/Paris"  # CAC,
+TIME_ZONE_BERLIN = "Europe/Berlin"  # DAX, TECDAX, MDAX, SDAX
+TIME_ZONE_JAKARTA = "Asia/Jakarta"  # LQ45
+TIME_ZONE_SELFDEFINED = "xxx"  # If neither of the above is your time zone, you should define it, and set USE_TIME_ZONE_SELFDEFINED 1.
+USE_TIME_ZONE_SELFDEFINED = 0  # 0 (default) or 1 (use the self defined)
+
+# parameters for data sources
+ALPACA_API_KEY = "xxx"  # your ALPACA_API_KEY
+ALPACA_API_SECRET = "xxx"  # your ALPACA_API_SECRET
+ALPACA_API_BASE_URL = "https://paper-api.alpaca.markets"  # alpaca url
+BINANCE_BASE_URL = "https://data.binance.vision/"  # binance url

finnlp/utils/config_tickers.py

@@ -0,0 +1,1429 @@
+SINGLE_TICKER = ["AAPL"]
+
+# Dow 30 constituents on 2021/10
+# check https://wrds-www.wharton.upenn.edu/ for U.S. index constituents (needs account)
+DOW_30_TICKER = [
+    "AXP",
+    "AMGN",
+    "AAPL",
+    "BA",
+    "CAT",
+    "CSCO",
+    "CVX",
+    "GS",
+    "HD",
+    "HON",
+    "IBM",
+    "INTC",
+    "JNJ",
+    "KO",
+    "JPM",
+    "MCD",
+    "MMM",
+    "MRK",
+    "MSFT",
+    "NKE",
+    "PG",
+    "TRV",
+    "UNH",
+    "CRM",
+    "VZ",
+    "V",
+    "WBA",
+    "WMT",
+    "DIS",
+    "DOW",
+]
+
+# Nasdaq 100 constituents on 2019/01
+NAS_100_TICKER = [
+    "AMGN",
+    "AAPL",
+    "AMAT",
+    "INTC",
+    "PCAR",
+    "PAYX",
+    "MSFT",
+    "ADBE",
+    "CSCO",
+    "XLNX",
+    "QCOM",
+    "COST",
+    "SBUX",
+    "FISV",
+    "CTXS",
+    "INTU",
+    "AMZN",
+    "EBAY",
+    "BIIB",
+    "CHKP",
+    "GILD",
+    "NLOK",
+    "CMCSA",
+    "FAST",
+    "ADSK",
+    "CTSH",
+    "NVDA",
+    "GOOGL",
+    "ISRG",
+    "VRTX",
+    "HSIC",
+    "BIDU",
+    "ATVI",
+    "ADP",
+    "ROST",
+    "ORLY",
+    "CERN",
+    "BKNG",
+    "MYL",
+    "MU",
+    "DLTR",
+    "ALXN",
+    "SIRI",
+    "MNST",
+    "AVGO",
+    "TXN",
+    "MDLZ",
+    "FB",
+    "ADI",
+    "WDC",
+    "REGN",
+    "LBTYK",
+    "VRSK",
+    "NFLX",
+    "TSLA",
+    "CHTR",
+    "MAR",
+    "ILMN",
+    "LRCX",
+    "EA",
+    "AAL",
+    "WBA",
+    "KHC",
+    "BMRN",
+    "JD",
+    "SWKS",
+    "INCY",
+    "PYPL",
+    "CDW",
+    "FOXA",
+    "MXIM",
+    "TMUS",
+    "EXPE",
+    "TCOM",
+    "ULTA",
+    "CSX",
+    "NTES",
+    "MCHP",
+    "CTAS",
+    "KLAC",
+    "HAS",
+    "JBHT",
+    "IDXX",
+    "WYNN",
+    "MELI",
+    "ALGN",
+    "CDNS",
+    "WDAY",
+    "SNPS",
+    "ASML",
+    "TTWO",
+    "PEP",
+    "NXPI",
+    "XEL",
+    "AMD",
+    "NTAP",
+    "VRSN",
+    "LULU",
+    "WLTW",
+    "UAL",
+]
+
+# SP 500 constituents at 2019
+SP_500_TICKER = [
+    "A",
+    "AAL",
+    "AAP",
+    "AAPL",
+    "ABBV",
+    "ABC",
+    "ABMD",
+    "ABT",
+    "ACN",
+    "ADBE",
+    "ADI",
+    "ADM",
+    "ADP",
+    "ADS",
+    "ADSK",
+    "AEE",
+    "AEP",
+    "AES",
+    "AFL",
+    "AGN",
+    "AIG",
+    "AIV",
+    "AIZ",
+    "AJG",
+    "AKAM",
+    "ALB",
+    "ALGN",
+    "ALK",
+    "ALL",
+    "ALLE",
+    "ALXN",
+    "AMAT",
+    "AMCR",
+    "AMD",
+    "AME",
+    "AMG",
+    "AMGN",
+    "AMP",
+    "AMT",
+    "AMZN",
+    "ANET",
+    "ANSS",
+    "ANTM",
+    "AON",
+    "AOS",
+    "APA",
+    "APD",
+    "APH",
+    "APTV",
+    "ARE",
+    "ARNC",
+    "ATO",
+    "ATVI",
+    "AVB",
+    "AVGO",
+    "AVY",
+    "AWK",
+    "AXP",
+    "AZO",
+    "BA",
+    "BAC",
+    "BAX",
+    "BBT",
+    "BBY",
+    "BDX",
+    "BEN",
+    "BF.B",
+    "BHGE",
+    "BIIB",
+    "BK",
+    "BKNG",
+    "BLK",
+    "BLL",
+    "BMY",
+    "BR",
+    "BRK.B",
+    "BSX",
+    "BWA",
+    "BXP",
+    "C",
+    "CAG",
+    "CAH",
+    "CAT",
+    "CB",
+    "CBOE",
+    "CBRE",
+    "CBS",
+    "CCI",
+    "CCL",
+    "CDNS",
+    "CE",
+    "CELG",
+    "CERN",
+    "CF",
+    "CFG",
+    "CHD",
+    "CHRW",
+    "CHTR",
+    "CI",
+    "CINF",
+    "CL",
+    "CLX",
+    "CMA",
+    "CMCSA",
+    "CME",
+    "CMG",
+    "CMI",
+    "CMS",
+    "CNC",
+    "CNP",
+    "COF",
+    "COG",
+    "COO",
+    "COP",
+    "COST",
+    "COTY",
+    "CPB",
+    "CPRI",
+    "CPRT",
+    "CRM",
+    "CSCO",
+    "CSX",
+    "CTAS",
+    "CTL",
+    "CTSH",
+    "CTVA",
+    "CTXS",
+    "CVS",
+    "CVX",
+    "CXO",
+    "D",
+    "DAL",
+    "DD",
+    "DE",
+    "DFS",
+    "DG",
+    "DGX",
+    "DHI",
+    "DHR",
+    "DIS",
+    "DISCK",
+    "DISH",
+    "DLR",
+    "DLTR",
+    "DOV",
+    "DOW",
+    "DRE",
+    "DRI",
+    "DTE",
+    "DUK",
+    "DVA",
+    "DVN",
+    "DXC",
+    "EA",
+    "EBAY",
+    "ECL",
+    "ED",
+    "EFX",
+    "EIX",
+    "EL",
+    "EMN",
+    "EMR",
+    "EOG",
+    "EQIX",
+    "EQR",
+    "ES",
+    "ESS",
+    "ETFC",
+    "ETN",
+    "ETR",
+    "EVRG",
+    "EW",
+    "EXC",
+    "EXPD",
+    "EXPE",
+    "EXR",
+    "F",
+    "FANG",
+    "FAST",
+    "FB",
+    "FBHS",
+    "FCX",
+    "FDX",
+    "FE",
+    "FFIV",
+    "FIS",
+    "FISV",
+    "FITB",
+    "FLIR",
+    "FLS",
+    "FLT",
+    "FMC",
+    "FOXA",
+    "FRC",
+    "FRT",
+    "FTI",
+    "FTNT",
+    "FTV",
+    "GD",
+    "GE",
+    "GILD",
+    "GIS",
+    "GL",
+    "GLW",
+    "GM",
+    "GOOG",
+    "GPC",
+    "GPN",
+    "GPS",
+    "GRMN",
+    "GS",
+    "GWW",
+    "HAL",
+    "HAS",
+    "HBAN",
+    "HBI",
+    "HCA",
+    "HCP",
+    "HD",
+    "HES",
+    "HFC",
+    "HIG",
+    "HII",
+    "HLT",
+    "HOG",
+    "HOLX",
+    "HON",
+    "HP",
+    "HPE",
+    "HPQ",
+    "HRB",
+    "HRL",
+    "HSIC",
+    "HST",
+    "HSY",
+    "HUM",
+    "IBM",
+    "ICE",
+    "IDXX",
+    "IEX",
+    "IFF",
+    "ILMN",
+    "INCY",
+    "INFO",
+    "INTC",
+    "INTU",
+    "IP",
+    "IPG",
+    "IPGP",
+    "IQV",
+    "IR",
+    "IRM",
+    "ISRG",
+    "IT",
+    "ITW",
+    "IVZ",
+    "JBHT",
+    "JCI",
+    "JEC",
+    "JEF",
+    "JKHY",
+    "JNJ",
+    "JNPR",
+    "JPM",
+    "JWN",
+    "K",
+    "KEY",
+    "KEYS",
+    "KHC",
+    "KIM",
+    "KLAC",
+    "KMB",
+    "KMI",
+    "KMX",
+    "KO",
+    "KR",
+    "KSS",
+    "KSU",
+    "L",
+    "LB",
+    "LDOS",
+    "LEG",
+    "LEN",
+    "LH",
+    "LHX",
+    "LIN",
+    "LKQ",
+    "LLY",
+    "LMT",
+    "LNC",
+    "LNT",
+    "LOW",
+    "LRCX",
+    "LUV",
+    "LW",
+    "LYB",
+    "M",
+    "MA",
+    "MAA",
+    "MAC",
+    "MAR",
+    "MAS",
+    "MCD",
+    "MCHP",
+    "MCK",
+    "MCO",
+    "MDLZ",
+    "MDT",
+    "MET",
+    "MGM",
+    "MHK",
+    "MKC",
+    "MKTX",
+    "MLM",
+    "MMC",
+    "MMM",
+    "MNST",
+    "MO",
+    "MOS",
+    "MPC",
+    "MRK",
+    "MRO",
+    "MS",
+    "MSCI",
+    "MSFT",
+    "MSI",
+    "MTB",
+    "MTD",
+    "MU",
+    "MXIM",
+    "MYL",
+    "NBL",
+    "NCLH",
+    "NDAQ",
+    "NEE",
+    "NEM",
+    "NFLX",
+    "NI",
+    "NKE",
+    "NKTR",
+    "NLSN",
+    "NOC",
+    "NOV",
+    "NRG",
+    "NSC",
+    "NTAP",
+    "NTRS",
+    "NUE",
+    "NVDA",
+    "NWL",
+    "NWS",
+    "O",
+    "OI",
+    "OKE",
+    "OMC",
+    "ORCL",
+    "ORLY",
+    "OXY",
+    "PAYX",
+    "PBCT",
+    "PCAR",
+    "PEG",
+    "PEP",
+    "PFE",
+    "PFG",
+    "PG",
+    "PGR",
+    "PH",
+    "PHM",
+    "PKG",
+    "PKI",
+    "PLD",
+    "PM",
+    "PNC",
+    "PNR",
+    "PNW",
+    "PPG",
+    "PPL",
+    "PRGO",
+    "PRU",
+    "PSA",
+    "PSX",
+    "PVH",
+    "PWR",
+    "PXD",
+    "PYPL",
+    "QCOM",
+    "QRVO",
+    "RCL",
+    "RE",
+    "REG",
+    "REGN",
+    "RF",
+    "RHI",
+    "RJF",
+    "RL",
+    "RMD",
+    "ROK",
+    "ROL",
+    "ROP",
+    "ROST",
+    "RSG",
+    "RTN",
+    "SBAC",
+    "SBUX",
+    "SCHW",
+    "SEE",
+    "SHW",
+    "SIVB",
+    "SJM",
+    "SLB",
+    "SLG",
+    "SNA",
+    "SNPS",
+    "SO",
+    "SPG",
+    "SPGI",
+    "SRE",
+    "STI",
+    "STT",
+    "STX",
+    "STZ",
+    "SWK",
+    "SWKS",
+    "SYF",
+    "SYK",
+    "SYMC",
+    "SYY",
+    "T",
+    "TAP",
+    "TDG",
+    "TEL",
+    "TFX",
+    "TGT",
+    "TIF",
+    "TJX",
+    "TMO",
+    "TMUS",
+    "TPR",
+    "TRIP",
+    "TROW",
+    "TRV",
+    "TSCO",
+    "TSN",
+    "TSS",
+    "TTWO",
+    "TWTR",
+    "TXN",
+    "TXT",
+    "UA",
+    "UAL",
+    "UDR",
+    "UHS",
+    "ULTA",
+    "UNH",
+    "UNM",
+    "UNP",
+    "UPS",
+    "URI",
+    "USB",
+    "UTX",
+    "V",
+    "VAR",
+    "VFC",
+    "VIAB",
+    "VLO",
+    "VMC",
+    "VNO",
+    "VRSK",
+    "VRSN",
+    "VRTX",
+    "VTR",
+    "VZ",
+    "WAB",
+    "WAT",
+    "WBA",
+    "WCG",
+    "WDC",
+    "WEC",
+    "WELL",
+    "WFC",
+    "WHR",
+    "WLTW",
+    "WM",
+    "WMB",
+    "WMT",
+    "WRK",
+    "WU",
+    "WY",
+    "WYNN",
+    "XEC",
+    "XEL",
+    "XLNX",
+    "XOM",
+    "XRAY",
+    "XRX",
+    "XYL",
+    "YUM",
+    "ZBH",
+    "ZION",
+    "ZTS",
+]
+
+# Hang Seng Index constituents at 2019/01
+HSI_50_TICKER = [
+    "0011.HK",
+    "0005.HK",
+    "0012.HK",
+    "0006.HK",
+    "0003.HK",
+    "0016.HK",
+    "0019.HK",
+    "0002.HK",
+    "0001.HK",
+    "0267.HK",
+    "0101.HK",
+    "0941.HK",
+    "0762.HK",
+    "0066.HK",
+    "0883.HK",
+    "2388.HK",
+    "0017.HK",
+    "0083.HK",
+    "0939.HK",
+    "0388.HK",
+    "0386.HK",
+    "3988.HK",
+    "2628.HK",
+    "1398.HK",
+    "2318.HK",
+    "3328.HK",
+    "0688.HK",
+    "0857.HK",
+    "1088.HK",
+    "0700.HK",
+    "0836.HK",
+    "1109.HK",
+    "1044.HK",
+    "1299.HK",
+    "0151.HK",
+    "1928.HK",
+    "0027.HK",
+    "2319.HK",
+    "0823.HK",
+    "1113.HK",
+    "1038.HK",
+    "2018.HK",
+    "0175.HK",
+    "0288.HK",
+    "1997.HK",
+    "2007.HK",
+    "2382.HK",
+    "1093.HK",
+    "1177.HK",
+    "2313.HK",
+]
+
+# SSE 50 Index constituents in 2019. www.csindex.com.cn, for SSE and CSI adjustments
+SSE_50_TICKER = [
+    "600000.XSHG",
+    "600036.XSHG",
+    "600104.XSHG",
+    "600030.XSHG",
+    "601628.XSHG",
+    "601166.XSHG",
+    "601318.XSHG",
+    "601328.XSHG",
+    "601088.XSHG",
+    "601857.XSHG",
+    "601601.XSHG",
+    "601668.XSHG",
+    "601288.XSHG",
+    "601818.XSHG",
+    "601989.XSHG",
+    "601398.XSHG",
+    "600048.XSHG",
+    "600028.XSHG",
+    "600050.XSHG",
+    "600519.XSHG",
+    "600016.XSHG",
+    "600887.XSHG",
+    "601688.XSHG",
+    "601186.XSHG",
+    "601988.XSHG",
+    "601211.XSHG",
+    "601336.XSHG",
+    "600309.XSHG",
+    "603993.XSHG",
+    "600690.XSHG",
+    "600276.XSHG",
+    "600703.XSHG",
+    "600585.XSHG",
+    "603259.XSHG",
+    "601888.XSHG",
+    "601138.XSHG",
+    "600196.XSHG",
+    "601766.XSHG",
+    "600340.XSHG",
+    "601390.XSHG",
+    "601939.XSHG",
+    "601111.XSHG",
+    "600029.XSHG",
+    "600019.XSHG",
+    "601229.XSHG",
+    "601800.XSHG",
+    "600547.XSHG",
+    "601006.XSHG",
+    "601360.XSHG",
+    "600606.XSHG",
+    "601319.XSHG",
+    "600837.XSHG",
+    "600031.XSHG",
+    "601066.XSHG",
+    "600009.XSHG",
+    "601236.XSHG",
+    "601012.XSHG",
+    "600745.XSHG",
+    "600588.XSHG",
+    "601658.XSHG",
+    "601816.XSHG",
+    "603160.XSHG",
+]
+
+# CSI 300 Index constituents at 2019
+CSI_300_TICKER = [
+    "600000.XSHG",
+    "600004.XSHG",
+    "600009.XSHG",
+    "600010.XSHG",
+    "600011.XSHG",
+    "600015.XSHG",
+    "600016.XSHG",
+    "600018.XSHG",
+    "600019.XSHG",
+    "600025.XSHG",
+    "600027.XSHG",
+    "600028.XSHG",
+    "600029.XSHG",
+    "600030.XSHG",
+    "600031.XSHG",
+    "600036.XSHG",
+    "600038.XSHG",
+    "600048.XSHG",
+    "600050.XSHG",
+    "600061.XSHG",
+    "600066.XSHG",
+    "600068.XSHG",
+    "600085.XSHG",
+    "600089.XSHG",
+    "600104.XSHG",
+    "600109.XSHG",
+    "600111.XSHG",
+    "600115.XSHG",
+    "600118.XSHG",
+    "600170.XSHG",
+    "600176.XSHG",
+    "600177.XSHG",
+    "600183.XSHG",
+    "600188.XSHG",
+    "600196.XSHG",
+    "600208.XSHG",
+    "600219.XSHG",
+    "600221.XSHG",
+    "600233.XSHG",
+    "600271.XSHG",
+    "600276.XSHG",
+    "600297.XSHG",
+    "600299.XSHG",
+    "600309.XSHG",
+    "600332.XSHG",
+    "600340.XSHG",
+    "600346.XSHG",
+    "600352.XSHG",
+    "600362.XSHG",
+    "600369.XSHG",
+    "600372.XSHG",
+    "600383.XSHG",
+    "600390.XSHG",
+    "600398.XSHG",
+    "600406.XSHG",
+    "600436.XSHG",
+    "600438.XSHG",
+    "600482.XSHG",
+    "600487.XSHG",
+    "600489.XSHG",
+    "600498.XSHG",
+    "600516.XSHG",
+    "600519.XSHG",
+    "600522.XSHG",
+    "600547.XSHG",
+    "600570.XSHG",
+    "600583.XSHG",
+    "600585.XSHG",
+    "600588.XSHG",
+    "600606.XSHG",
+    "600637.XSHG",
+    "600655.XSHG",
+    "600660.XSHG",
+    "600674.XSHG",
+    "600690.XSHG",
+    "600703.XSHG",
+    "600705.XSHG",
+    "600741.XSHG",
+    "600745.XSHG",
+    "600760.XSHG",
+    "600795.XSHG",
+    "600809.XSHG",
+    "600837.XSHG",
+    "600848.XSHG",
+    "600867.XSHG",
+    "600886.XSHG",
+    "600887.XSHG",
+    "600893.XSHG",
+    "600900.XSHG",
+    "600919.XSHG",
+    "600926.XSHG",
+    "600928.XSHG",
+    "600958.XSHG",
+    "600968.XSHG",
+    "600977.XSHG",
+    "600989.XSHG",
+    "600998.XSHG",
+    "600999.XSHG",
+    "601006.XSHG",
+    "601009.XSHG",
+    "601012.XSHG",
+    "601018.XSHG",
+    "601021.XSHG",
+    "601066.XSHG",
+    "601077.XSHG",
+    "601088.XSHG",
+    "601100.XSHG",
+    "601108.XSHG",
+    "601111.XSHG",
+    "601117.XSHG",
+    "601138.XSHG",
+    "601155.XSHG",
+    "601162.XSHG",
+    "601166.XSHG",
+    "601169.XSHG",
+    "601186.XSHG",
+    "601198.XSHG",
+    "601211.XSHG",
+    "601212.XSHG",
+    "601216.XSHG",
+    "601225.XSHG",
+    "601229.XSHG",
+    "601231.XSHG",
+    "601236.XSHG",
+    "601238.XSHG",
+    "601288.XSHG",
+    "601298.XSHG",
+    "601318.XSHG",
+    "601319.XSHG",
+    "601328.XSHG",
+    "601336.XSHG",
+    "601360.XSHG",
+    "601377.XSHG",
+    "601390.XSHG",
+    "601398.XSHG",
+    "601555.XSHG",
+    "601577.XSHG",
+    "601600.XSHG",
+    "601601.XSHG",
+    "601607.XSHG",
+    "601618.XSHG",
+    "601628.XSHG",
+    "601633.XSHG",
+    "601658.XSHG",
+    "601668.XSHG",
+    "601669.XSHG",
+    "601688.XSHG",
+    "601698.XSHG",
+    "601727.XSHG",
+    "601766.XSHG",
+    "601788.XSHG",
+    "601800.XSHG",
+    "601808.XSHG",
+    "601816.XSHG",
+    "601818.XSHG",
+    "601828.XSHG",
+    "601838.XSHG",
+    "601857.XSHG",
+    "601877.XSHG",
+    "601878.XSHG",
+    "601881.XSHG",
+    "601888.XSHG",
+    "601898.XSHG",
+    "601899.XSHG",
+    "601901.XSHG",
+    "601916.XSHG",
+    "601919.XSHG",
+    "601933.XSHG",
+    "601939.XSHG",
+    "601985.XSHG",
+    "601988.XSHG",
+    "601989.XSHG",
+    "601992.XSHG",
+    "601997.XSHG",
+    "601998.XSHG",
+    "603019.XSHG",
+    "603156.XSHG",
+    "603160.XSHG",
+    "603259.XSHG",
+    "603260.XSHG",
+    "603288.XSHG",
+    "603369.XSHG",
+    "603501.XSHG",
+    "603658.XSHG",
+    "603799.XSHG",
+    "603833.XSHG",
+    "603899.XSHG",
+    "603986.XSHG",
+    "603993.XSHG",
+    "000001.XSHE",
+    "000002.XSHE",
+    "000063.XSHE",
+    "000066.XSHE",
+    "000069.XSHE",
+    "000100.XSHE",
+    "000157.XSHE",
+    "000166.XSHE",
+    "000333.XSHE",
+    "000338.XSHE",
+    "000425.XSHE",
+    "000538.XSHE",
+    "000568.XSHE",
+    "000596.XSHE",
+    "000625.XSHE",
+    "000627.XSHE",
+    "000651.XSHE",
+    "000656.XSHE",
+    "000661.XSHE",
+    "000671.XSHE",
+    "000703.XSHE",
+    "000708.XSHE",
+    "000709.XSHE",
+    "000723.XSHE",
+    "000725.XSHE",
+    "000728.XSHE",
+    "000768.XSHE",
+    "000776.XSHE",
+    "000783.XSHE",
+    "000786.XSHE",
+    "000858.XSHE",
+    "000860.XSHE",
+    "000876.XSHE",
+    "000895.XSHE",
+    "000938.XSHE",
+    "000961.XSHE",
+    "000963.XSHE",
+    "000977.XSHE",
+    "001979.XSHE",
+    "002001.XSHE",
+    "002007.XSHE",
+    "002008.XSHE",
+    "002024.XSHE",
+    "002027.XSHE",
+    "002032.XSHE",
+    "002044.XSHE",
+    "002050.XSHE",
+    "002120.XSHE",
+    "002129.XSHE",
+    "002142.XSHE",
+    "002146.XSHE",
+    "002153.XSHE",
+    "002157.XSHE",
+    "002179.XSHE",
+    "002202.XSHE",
+    "002230.XSHE",
+    "002236.XSHE",
+    "002241.XSHE",
+    "002252.XSHE",
+    "002271.XSHE",
+    "002304.XSHE",
+    "002311.XSHE",
+    "002352.XSHE",
+    "002371.XSHE",
+    "002410.XSHE",
+    "002415.XSHE",
+    "002422.XSHE",
+    "002456.XSHE",
+    "002460.XSHE",
+    "002463.XSHE",
+    "002466.XSHE",
+    "002468.XSHE",
+    "002475.XSHE",
+    "002493.XSHE",
+    "002508.XSHE",
+    "002555.XSHE",
+    "002558.XSHE",
+    "002594.XSHE",
+    "002601.XSHE",
+    "002602.XSHE",
+    "002607.XSHE",
+    "002624.XSHE",
+    "002673.XSHE",
+    "002714.XSHE",
+    "002736.XSHE",
+    "002739.XSHE",
+    "002773.XSHE",
+    "002841.XSHE",
+    "002916.XSHE",
+    "002938.XSHE",
+    "002939.XSHE",
+    "002945.XSHE",
+    "002958.XSHE",
+    "003816.XSHE",
+    "300003.XSHE",
+    "300014.XSHE",
+    "300015.XSHE",
+    "300033.XSHE",
+    "300059.XSHE",
+    "300122.XSHE",
+    "300124.XSHE",
+    "300136.XSHE",
+    "300142.XSHE",
+    "300144.XSHE",
+    "300347.XSHE",
+    "300408.XSHE",
+    "300413.XSHE",
+    "300433.XSHE",
+    "300498.XSHE",
+    "300601.XSHE",
+    "300628.XSHE",
+]
+
+# CAC 40 constituents at 2019/01
+# Check https://www.bnains.org/archives/histocac/compocac.php for CAC 40 constituents
+CAC_40_TICKER = [
+    "AC.PA",
+    "AI.PA",
+    "AIR.PA",
+    "MT.AS",
+    "ATO.PA",
+    "CS.PA",
+    "BNP.PA",
+    "EN.PA",
+    "CAP.PA",
+    "CA.PA",
+    "ACA.PA",
+    "BN.PA",
+    "DSY.PA",
+    "ENGI.PA",
+    "EL.PA",
+    "RMS.PA",
+    "KER.PA",
+    "OR.PA",
+    "LR.PA",
+    "MC.PA",
+    "ML.PA",
+    "ORA.PA",
+    "RI.PA",
+    "PUGOY",
+    "PUB.PA",
+    "RNO.PA",
+    "SAF.PA",
+    "SGO.PA",
+    "SAN.PA",
+    "SU.PA",
+    "GLE.PA",
+    "SW.PA",
+    "STM.PA",
+    "FTI.PA",
+    "FP.PA",
+    "URW.AS",
+    "FR.PA",
+    "VIE.PA",
+    "DG.PA",
+    "VIV.PA",
+]
+
+# DAX 30 constituents on 2021/02
+DAX_30_TICKER = [
+    "DHER.DE",
+    "RWE.DE",
+    "FRE.DE",
+    "MTX.DE",
+    "MRK.DE",
+    "LIN.DE",
+    "ALV.DE",
+    "VNA.DE",
+    "EOAN.DE",
+    "HEN3.DE",
+    "DAI.DE",
+    "DB1.DE",
+    "DPW.DE",
+    "DWNI.DE",
+    "BMW.DE",
+    "DTE.DE",
+    "VOW3.DE",
+    "MUV2.DE",
+    "1COV.DE",
+    "SAP.DE",
+    "FME.DE",
+    "BAS.DE",
+    "BAYN.DE",
+    "BEI.DE",
+    "CON.DE",
+    "SIE.DE",
+    "ADS.DE",
+    "HEI.DE",
+    "DBK.DE",
+    "IFX.DE",
+]
+
+# TecDAX constituents at 2021/02
+TECDAX_TICKER = [
+    "ADV.DE",
+    "AFX.DE",
+    "AM3D.DE",
+    "BC8.DE",
+    "COK.DE",
+    "DLG.DE",
+    "DRI.DE",
+    "DRW3.DE",
+    "EVT.DE",
+    "FNTN.DE",
+    "GFT.DE",
+    "JEN.DE",
+    "MDG1.DE",
+    "MOR.DE",
+    "NDX1.DE",
+    "NEM.DE",
+    "O2D.DE",
+    "PFV.DE",
+    "QIA.DE",
+    "RIB.DE",
+    "S92.DE",
+    "SANT.DE",
+    "SOW.DE",
+    "SRT3.DE",
+    "UTDI.DE",
+    "WAF.DE",
+    "WDI.DE",
+]
+
+# MDAX 50 constituents at 2021/02
+MDAX_50_TICKER = [
+    "1COV.DE",
+    "AIR.DE",
+    "AOX.DE",
+    "ARL.DE",
+    "BNR.DE",
+    "BOSS.DE",
+    "DEQ.DE",
+    "DUE.DE",
+    "DWNI.DE",
+    "EVD.DE",
+    "EVK.DE",
+    "FIE.DE",
+    "FPE3.DE",
+    "FRA.DE",
+    "G1A.DE",
+    "GBF.DE",
+    "GXI.DE",
+    "HLE.DE",
+    "HNR1.DE",
+    "HOT.DE",
+    "JUN3.DE",
+    "KGX.DE",
+    "KRN.DE",
+    "LEG.DE",
+    "LEO.DE",
+    "LXS.DE",
+    "MTX.DE",
+    "NDA.DE",
+    "NOEJ.DE",
+    "OSR.DE",
+    "PBB.DE",
+    "RAA.DE",
+    "RHM.DE",
+    "RRTL.DE",
+    "SAX.DE",
+    "SDF.DE",
+    "SHA.DE",
+    "SNH.DE",
+    "SY1.DE",
+    "SZG.DE",
+    "SZU.DE",
+    "TEG.DE",
+    "TLX.DE",
+    "UN01.DE",
+    "WCH.DE",
+    "ZAL.DE",
+]
+
+# SDAX 50 constituents at 2021/02
+SDAX_50_TICKER = [
+    "AAD.DE",
+    "ACX.DE",
+    "ADJ.DE",
+    "ADL.DE",
+    "BDT.DE",
+    "BIO3.DE",
+    "BVB.DE",
+    "BYW6.DE",
+    "CWC.DE",
+    "DBAN.DE",
+    "DEZ.DE",
+    "DIC.DE",
+    "G24.DE",
+    "GIL.DE",
+    "GLJ.DE",
+    "GMM.DE",
+    "HBH.DE",
+    "HDD.DE",
+    "HHFA.DE",
+    "HLAG.DE",
+    "HYQ.DE",
+    "INH.DE",
+    "KCO.DE",
+    "KWS.DE",
+    "PUM.DE",
+    "RHK.DE",
+    "SFQ.DE",
+    "SGL.DE",
+    "SIX2.DE",
+    "SKB.DE",
+    "STM.DE",
+    "TC1.DE",
+    "TLG.DE",
+    "TTK.DE",
+    "VOS.DE",
+    "WAC.DE",
+    "WCMK.DE",
+    "WSU.DE",
+    "WUW.DE",
+    "ZIL2.DE",
+    "ZO1.DE",
+]
+
+# LQ45 constituents at 2021/10
+LQ45_TICKER = [
+    "ACES.JK",
+    "ADRO.JK",
+    "AKRA.JK",
+    "ANTM.JK",
+    "ASII.JK",
+    "BBCA.JK",
+    "BBNI.JK",
+    "BBRI.JK",
+    "BBTN.JK",
+    "BMRI.JK",
+    "BRPT.JK",
+    "BSDE.JK",
+    "CPIN.JK",
+    "ERAA.JK",
+    "EXCL.JK",
+    "GGRM.JK",
+    "HMSP.JK",
+    "ICBP.JK",
+    "INCO.JK",
+    "INDF.JK",
+    "INKP.JK",
+    "INTP.JK",
+    "ITMG.JK",
+    "JPFA.JK",
+    "JSMR.JK",
+    "KLBF.JK",
+    "MDKA.JK",
+    "MEDC.JK",
+    "MIKA.JK",
+    "MNCN.JK",
+    "PGAS.JK",
+    "PTBA.JK",
+    "PTPP.JK",
+    "PWON.JK",
+    "SMGR.JK",
+    "SMRA.JK",
+    "TBIG.JK",
+    "TINS.JK",
+    "TKIM.JK",
+    "TLKM.JK",
+    "TOWR.JK",
+    "TPIA.JK",
+    "UNTR.JK",
+    "UNVR.JK",
+    "WIKA.JK",
+]
+
+# SRI-KEHATI.JK - Sustainable Responsible Investm
+SRI_KEHATI_TICKER = [
+    "AALI.JK",
+    "ADHI.JK",
+    "ASII.JK",
+    "BBCA.JK",
+    "BBNI.JK",
+    "BBRI.JK",
+    "BBTN.JK",
+    "BMRI.JK",
+    "BSDE.JK",
+    "INDF.JK",
+    "JPFA.JK",
+    "JSMR.JK",
+    "KLBF.JK",
+    "PGAS.JK",
+    "PJAA.JK",
+    "PPRO.JK",
+    "SIDO.JK",
+    "SMGR.JK",
+    "TINS.JK",
+    "TLKM.JK",
+    "UNTR.JK",
+    "UNVR.JK",
+    "WIKA.JK",
+    "WSKT.JK",
+    "WTON.JK",
+]
+
+# FX Ticker
+FX_TICKER = [
+    "AUDCAD=X",
+    "AUDCHF=X",
+    "AUDJPY=X",
+    "AUDNZD=X",
+    "AUDSGD=X",
+    "AUDUSD=X",
+    "AUDUSD=X",
+    "AUDUSD=X",
+    "AUDUSD=X",
+    "AUDUSD=X",
+    "AUDUSD=X",
+    "AUDUSD=X",
+    "CADCHF=X",
+    "CADHKD=X",
+    "CADJPY=X",
+    "CHFJPY=X",
+    "CHFSGD=X",
+    "EURAUD=X",
+    "EURCAD=X",
+    "EURCHF=X",
+    "EURCHF=X",
+    "EURCHF=X",
+    "EURCZK=X",
+    "EURGBP=X",
+    "EURHKD=X",
+    "EURHUF=X",
+    "EURJPY=X",
+    "EURNOK=X",
+    "EURNZD=X",
+    "EURPLN=X",
+    "EURRUB=X",
+    "EURSEK=X",
+    "EURSGD=X",
+    "EURTRY=X",
+    "EURTRY=X",
+    "EURUSD=X",
+    "GBPAUD=X",
+    "GBPAUD=X",
+    "GBPAUD=X",
+    "GBPCAD=X",
+    "GBPCHF=X",
+    "GBPJPY=X",
+    "GBPNZD=X",
+    "GBPUSD=X",
+    "HKDJPY=X",
+    "NZDCAD=X",
+    "NZDCHF=X",
+    "NZDJPY=X",
+    "NZDUSD=X",
+    "SGDJPY=X",
+    "TRYJPY=X",
+    "USDCAD=X",
+    "USDCHF=X",
+    "USDCNH=X",
+    "USDCZK=X",
+    "USDHKD=X",
+    "USDHUF=X",
+    "USDILS=X",
+    "USDJPY=X",
+    "USDMXN=X",
+    "USDNOK=X",
+    "USDPLN=X",
+    "USDRON=X",
+    "USDRUB=X",
+    "USDSEK=X",
+    "USDSGD=X",
+    "USDTHB=X",
+    "USDTRY=X",
+    "USDZAR=X",
+    "XAGUSD=X",
+    "XAUUSD=X",
+    "ZARJPY=X",
+    "EURDKK=X",
+]

finnlp/utils/data_processor.py

@@ -0,0 +1,247 @@
+import os
+import pickle
+from typing import List
+
+import numpy as np
+import pandas as pd
+
+
+class DataProcessor:
+    def __init__(
+        self,
+        data_source: str,
+        start_date: str,
+        end_date: str,
+        time_interval: str,
+        **kwargs,
+    ):
+        self.data_source = data_source
+        self.start_date = start_date
+        self.end_date = end_date
+        self.time_interval = time_interval
+        self.dataframe = pd.DataFrame()
+
+        if self.data_source == "akshare":
+            from meta.data_processors.akshare import Akshare
+
+            processor_dict = {self.data_source: Akshare}
+        elif self.data_source == "alpaca":
+            from meta.data_processors.alpaca import Alpaca
+
+            processor_dict = {self.data_source: Alpaca}
+        elif self.data_source == "alphavantage":
+            from meta.data_processors.alphavantage import Alphavantage
+
+            processor_dict = {self.data_source: Alphavantage}
+        elif self.data_source == "baostock":
+            from meta.data_processors.baostock import Baostock
+
+            processor_dict = {self.data_source: Baostock}
+        elif self.data_source == "binance":
+            from meta.data_processors.binance import Binance
+
+            processor_dict = {self.data_source: Binance}
+        elif self.data_source == "ccxt":
+            from meta.data_processors.ccxt import Ccxt
+
+            processor_dict = {self.data_source: Ccxt}
+        elif self.data_source == "iexcloud":
+            from meta.data_processors.iexcloud import Iexcloud
+
+            processor_dict = {self.data_source: Iexcloud}
+        elif self.data_source == "joinquant":
+            from meta.data_processors.joinquant import Joinquant
+
+            processor_dict = {self.data_source: Joinquant}
+        elif self.data_source == "quandl":
+            from meta.data_processors.quandl import Quandl
+
+            processor_dict = {self.data_source: Quandl}
+        elif self.data_source == "quantconnect":
+            from meta.data_processors.quantconnect import Quantconnect
+
+            processor_dict = {self.data_source: Quantconnect}
+        elif self.data_source == "ricequant":
+            from meta.data_processors.ricequant import Ricequant
+
+            processor_dict = {self.data_source: Ricequant}
+        elif self.data_source == "tushare":
+            from meta.data_processors.tushare import Tushare
+
+            processor_dict = {self.data_source: Tushare}
+        elif self.data_source == "wrds":
+            from meta.data_processors.wrds import Wrds
+
+            processor_dict = {self.data_source: Wrds}
+        elif self.data_source == "yahoofinance":
+            from meta.data_processors.yahoofinance import Yahoofinance
+
+            processor_dict = {self.data_source: Yahoofinance}
+        else:
+            print(f"{self.data_source} is NOT supported yet.")
+
+        try:
+            self.processor = processor_dict.get(self.data_source)(
+                data_source, start_date, end_date, time_interval, **kwargs
+            )
+            print(f"{self.data_source} successfully connected")
+        except:
+            raise ValueError(
+                f"Please input correct account info for {self.data_source}!"
+            )
+
+    def download_data(self, ticker_list):
+        self.processor.download_data(ticker_list=ticker_list)
+        self.dataframe = self.processor.dataframe
+
+    def clean_data(self):
+        self.processor.dataframe = self.dataframe
+        self.processor.clean_data()
+        self.dataframe = self.processor.dataframe
+
+    def add_technical_indicator(
+        self, tech_indicator_list: List[str], select_stockstats_talib: int = 0
+    ):
+        self.tech_indicator_list = tech_indicator_list
+        self.processor.add_technical_indicator(
+            tech_indicator_list, select_stockstats_talib
+        )
+        self.dataframe = self.processor.dataframe
+
+    def add_turbulence(self):
+        self.processor.add_turbulence()
+        self.dataframe = self.processor.dataframe
+
+    def add_vix(self):
+        self.processor.add_vix()
+        self.dataframe = self.processor.dataframe
+
+    def df_to_array(self, if_vix: bool) -> np.array:
+        price_array, tech_array, turbulence_array = self.processor.df_to_array(
+            self.tech_indicator_list, if_vix
+        )
+        # fill nan with 0 for technical indicators
+        tech_nan_positions = np.isnan(tech_array)
+        tech_array[tech_nan_positions] = 0
+
+        return price_array, tech_array, turbulence_array
+
+    def data_split(self, df, start, end, target_date_col="time"):
+        """
+        split the dataset into training or testing using date
+        :param data: (df) pandas dataframe, start, end
+        :return: (df) pandas dataframe
+        """
+        data = df[(df[target_date_col] >= start) & (df[target_date_col] < end)]
+        data = data.sort_values([target_date_col, "tic"], ignore_index=True)
+        data.index = data[target_date_col].factorize()[0]
+        return data
+
+    def fillna(self):
+        self.processor.dataframe = self.dataframe
+        self.processor.fillna()
+        self.dataframe = self.processor.dataframe
+
+    def run(
+        self,
+        ticker_list: str,
+        technical_indicator_list: List[str],
+        if_vix: bool,
+        cache: bool = False,
+        select_stockstats_talib: int = 0,
+    ):
+        if self.time_interval == "1s" and self.data_source != "binance":
+            raise ValueError(
+                "Currently 1s interval data is only supported with 'binance' as data source"
+            )
+
+        cache_filename = (
+            "_".join(
+                ticker_list
+                + [
+                    self.data_source,
+                    self.start_date,
+                    self.end_date,
+                    self.time_interval,
+                ]
+            )
+            + ".pickle"
+        )
+        cache_dir = "./cache"
+        cache_path = os.path.join(cache_dir, cache_filename)
+
+        if cache and os.path.isfile(cache_path):
+            print(f"Using cached file {cache_path}")
+            self.tech_indicator_list = technical_indicator_list
+            with open(cache_path, "rb") as handle:
+                self.processor.dataframe = pickle.load(handle)
+        else:
+            self.download_data(ticker_list)
+            self.clean_data()
+            if cache:
+                if not os.path.exists(cache_dir):
+                    os.mkdir(cache_dir)
+                with open(cache_path, "wb") as handle:
+                    pickle.dump(
+                        self.dataframe,
+                        handle,
+                        protocol=pickle.HIGHEST_PROTOCOL,
+                    )
+
+        self.add_technical_indicator(technical_indicator_list, select_stockstats_talib)
+        if if_vix:
+            self.add_vix()
+        price_array, tech_array, turbulence_array = self.df_to_array(if_vix)
+        tech_nan_positions = np.isnan(tech_array)
+        tech_array[tech_nan_positions] = 0
+
+        return price_array, tech_array, turbulence_array
+
+
+def test_joinquant():
+    # TRADE_START_DATE = "2019-09-01"
+    TRADE_START_DATE = "2020-09-01"
+    TRADE_END_DATE = "2021-09-11"
+
+    # supported time interval: '1m', '5m', '15m', '30m', '60m', '120m', '1d', '1w', '1M'
+    TIME_INTERVAL = "1d"
+    TECHNICAL_INDICATOR = [
+        "macd",
+        "boll_ub",
+        "boll_lb",
+        "rsi_30",
+        "dx_30",
+        "close_30_sma",
+        "close_60_sma",
+    ]
+
+    kwargs = {"username": "xxx", "password": "xxx"}
+    p = DataProcessor(
+        data_source="joinquant",
+        start_date=TRADE_START_DATE,
+        end_date=TRADE_END_DATE,
+        time_interval=TIME_INTERVAL,
+        **kwargs,
+    )
+
+    ticker_list = ["000612.XSHE", "601808.XSHG"]
+
+    p.download_data(ticker_list=ticker_list)
+
+    p.clean_data()
+    p.add_turbulence()
+    p.add_technical_indicator(TECHNICAL_INDICATOR)
+    p.add_vix()
+
+    price_array, tech_array, turbulence_array = p.run(
+        ticker_list, TECHNICAL_INDICATOR, if_vix=False, cache=True
+    )
+    pass
+
+
+# if __name__ == "__main__":
+#     # test_joinquant()
+#     # test_binance()
+#     # test_yahoofinance()
+#     test_baostock()
+#     # test_quandl()

finnlp/utils/get_proxy.py

@@ -0,0 +1,117 @@
+import requests
+import parsel
+from lxml import etree
+from tqdm import tqdm
+import time
+import re
+
+def check_china_ips(proxies_list):
+    """检测ip的方法"""
+    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36'}
+    
+    can_use = []
+    for proxy in tqdm(proxies_list, desc = "Checking ips"):
+        try:
+            response = requests.get('http://www.baidu.com', headers=headers, proxies=proxy, timeout=1)  # 超时报错
+            if response.status_code == 200:
+                can_use.append(proxy)
+        except Exception as error:
+            # print(error)
+            pass
+    return can_use
+
+def check_us_ips(proxies_list):
+    """检测ip的方法"""
+    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36'}
+
+    can_use = []
+    for proxy in tqdm(proxies_list, desc = "Checking ips"):
+        try:
+            response = requests.get('http://www.google.com', headers=headers, proxies=proxy, timeout=1)  # 超时报错
+            if response.status_code == 200:
+                can_use.append(proxy)
+        except Exception as error:
+            # print(error)
+            pass
+    return can_use
+
+def get_china_free_proxy(pages = 10):
+    proxies_list = []
+    for page in tqdm(range(1, pages+1), desc = "Gathering free ips by pages..."):
+
+        base_url = f'https://www.kuaidaili.com/free/inha/{page}'
+        headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36'}
+        success = False
+        while not success:
+            try:
+                response = requests.get(base_url, headers=headers)
+                data = response.text
+                res = etree.HTML(data)
+                trs = res.xpath("/html/body/div[1]/div[4]/div[2]/div[2]/div[2]/table/tbody/tr")
+                if len(trs)!=0:
+                    success = True
+                    for tr in trs:
+                        proxies_dict = {}
+                        http_type = tr.xpath('./td[4]/text()')[0]
+                        ip_num = tr.xpath('./td[1]/text()')[0]
+                        port_num = tr.xpath('./td[2]/text()')[0]
+                        proxies_dict[http_type] = ip_num + ':' + port_num
+                        proxies_list.append(proxies_dict)
+                else:
+                    time.delay(0.01)
+      
+            except:
+                pass
+
+    can_use = check_china_ips(proxies_list)
+
+    print(f'获取到的代理ip数量: {len(proxies_list)} 。Get proxy ips: {len(proxies_list)}.')
+    print(f'能用的代理数量： {len(can_use)}。Usable proxy ips: {len(can_use)}.' )
+
+    return can_use
+
+def get_us_free_proxy(pages = 10):
+    url = "https://openproxy.space/list/http"
+    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36'}
+    response = requests.get(url, headers=headers)
+    if response.status_code != 200:
+        print("Connection Error. Please make sure that your computer now have the access to Google.com")
+    res = etree.HTML(response.text)
+    http_type = "HTTP"
+    proxies_list = []
+
+    scripts = res.xpath("//script")
+    content = scripts[3].xpath(".//text()")
+    pattern = re.compile('LIST",data:(.+),added:')
+    result_list = pattern.findall(content[0])
+    result_list = result_list[0].strip("[{").strip("}]").split("},{")
+
+    for result in result_list:
+        pattern = re.compile('\[(.+)\]')
+        result = pattern.findall(result)
+        result = result[0].split(",")
+        result = [r.strip("\"") for r in result]
+        for ip in result:
+            proxies_list.append(
+                {http_type: ip}
+            )
+    total = pages* 15
+    proxies_list = proxies_list[:total] 
+    can_use = check_us_ips(proxies_list)
+    print(f'Get proxy ips: {len(proxies_list)}.')
+    print(f'Usable proxy ips: {len(can_use)}.' )
+
+    return can_use
+
+class Kuaidaili:
+    def __init__(self, tunnel, username, password):
+        self.tunnel = tunnel
+        self.username = username
+        self.password = password
+
+    def get_kuaidaili_tunnel_proxy(self):
+        proxies = {
+            "http": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": self.username, "pwd": self.password, "proxy": self.tunnel},
+            "https": "http://%(user)s:%(pwd)s@%(proxy)s/" % {"user": self.username, "pwd": self.password, "proxy": self.tunnel}
+        }
+        return proxies