MSDN磁力链爬虫

#!/usr/bin/python
# -*- coding: utf-8 -*-
import re
import json
import requests

API_MSDN_INDEX = 'https://msdn.itellyou.cn/'

API_INDEX = 'https://msdn.itellyou.cn/Index/GetCategory'

API_GET_LANG = 'https://msdn.itellyou.cn/Index/GetLang'

API_GET_LIST = 'https://msdn.itellyou.cn/Index/GetList'

API_GET_PRODUCT = 'https://msdn.itellyou.cn/Index/GetProduct'

headers = {
'Referer':'https://msdn.itellyou.cn/',
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.63 Safari/537.36',
'x-csrf-token':'CfDJ8Pkxne5b6ctAqJ7E_I5QAwlj_jfdNMW73lm7EXmK-wDUYnSG5jjFP_h5Wu-c9K-fzc0Ak5mgWc2sZlxXD0pvLako3f1eZ4l93SGD8djxN578BF7z7RUY1JMHihxkeYSmU6SLTlpilXTANrU2GCmagbE',
'cookie': 'UM_distinctid=17b96ee341b1ca-07d12fbf3a44a8-c343365-1fa400-17b96ee341c473; _ga=GA1.2.869490056.1630324799; .AspNetCore.Antiforgery.kC_Kc8he0KM=CfDJ8Pkxne5b6ctAqJ7E_I5QAwmrDvKmuf5nZKhFFl4mStgoRtDUvVyMMP41rRppdfK2HGaS4Pb5quYbQVc35oV2YR0oHszJ2cZre9tSd0pPSOJSk0QqE_qf6ENkEBRtBGRu4IWPGLrfIUZFOPAhHcMXP78; _gid=GA1.2.41224256.1632673751; Hm_lvt_8688ca4bc18cbc647c9c68fdaef6bc24=1630324799,1632673751; CNZZDATA1605814=cnzz_eid%3D294278726-1630317960-https%253A%252F%252Fwww.baidu.com%252F%26ntime%3D1632708437; Hm_lpvt_8688ca4bc18cbc647c9c68fdaef6bc24=1632718285'
}

RESULT = {'data':[]} 
def get_product(id):
    r = requests.post(API_GET_PRODUCT, headers=headers, data={'id':id})
    #print(r.text)
    if r.status_code == requests.codes.ok:
            item = r.json().get('result')
            print('文件名:%s' % item.get('filename'))
            #print 'PostData:%s' % item.get('PostDateString')
            print('SHA1校验值:%s' % item.get('sha1'))
            print('文件大小:%s' % item.get('size'))
            print('下载链接:%s' % item.get('download'))
            return item


def get_list(id, lang_id):
    r = requests.post(API_GET_LIST, headers=headers, data={'id':id, 'lang':lang_id, 'filter':'true'})
    if r.status_code == requests.codes.ok:
        product_list = []
        for item in r.json().get('result'):
            product_info = get_product(item.get('id'))
            product_list.append(product_info)
        return product_list

def get_lang(id):
    r = requests.post(API_GET_LANG, headers=headers, data={'id':id})
    if r.status_code == requests.codes.ok:
            lang_list = []
            for lang in r.json().get('result'):
                print("语言:",lang.get('lang'))
                info = {'lang':lang.get('lang'), 'product_list':get_list(id,lang.get('id'))}
                lang_list.append(info)
            return lang_list


def get_download_list(category_id):
    r = requests.post(API_INDEX, headers=headers, data={'id':category_id})
    #print(r.text)
    if r.status_code == requests.codes.ok:
            for item in r.json():
                #print('System Name: %s'% item.get('name'))
                system_info = get_lang(item.get('id'))
                system_info = {'name':item.get('name'), 'lang_list':system_info}
                RESULT['data'].append(system_info)
            #print('finishied!!!')
            return RESULT


if __name__ == '__main__':
    json_buffer = get_download_list('7AB5F0CB-7607-4BBE-9E88-50716DC43DE6')
    #print(json_buffer)
    #json.dump(json_buffer,f)