python Requsets Download code for open source Web site of with indexed data

Environmental construction

python 3.x
requests packet
re packet
gooey package (for visualization)

Code

import requests
import re
import os
from gooey import Gooey, GooeyParser
import time

s = requests.Session()

def judgeTypeOfPath(name):
    '''
     Determine whether the path is a file or a folder 
      :param name:  Path name 
      :return:True-> Documents ;False-> Folder 
    '''
    if name[-1] == '/':
        return False
    else:
        return True


def makeDirOfPath(path):
    '''
     Create a folder 
    :param path:  Folder name and path 
    :return: True-> Successful creation ;False-> Failed to create 
    '''
    if not os.path.isdir(path):
        os.mkdir(path)
    if not os.path.isdir(path):
        return False
    return True

def getPath(url):
    '''
     Get a list of page paths 
    :param url:  Current Page Path 
    :return:  Path list 
    '''
    baseResponse = s.get(url=url, stream=True,verify=False).text
    listOfDirOrFilesTemp = re.findall(r'<li><a href=".*?" rel="external nofollow" >', baseResponse)
    listOfDirOrFiles = []
    for i in range(len(listOfDirOrFilesTemp)):
        listOfDirOrFiles.append(listOfDirOrFilesTemp[i].split("\"")[1])
    return listOfDirOrFiles[1:len(listOfDirOrFiles) + 1]

def rfSearch(listOfPath,url, nowPath):
    '''
     Recursively find directories and paths , And download the file 
    :param listOfPath:  List of files and folders in the current directory 
    :param nowPath:  Path on which you are now 
    :return:
    '''
    newList = listOfPath[:]
    if not newList:
        return
    for i in range(len(newList)):
        if not judgeTypeOfPath(newList[i]):
            u = nowPath + newList[i][0:len(newList[i])]
            makeDirOfPath(u)
            tempPath=nowPath + newList[i][0:len(newList[i])+1]
            tempUrl=url+newList[i][0:len(newList[i])+1]
            u=getPath(tempUrl)
            rfSearch(u,tempUrl,tempPath)
        else:
            print(f' Start downloading {newList[i]}...')
            t1=time.time()
            u = nowPath + newList[i]
            m=url+newList[i]
            if not os.path.exists(u):
                r = s.get(m, stream=True,verify=False)
                f = open(u, "wb")
                for chunk in r.iter_content(chunk_size=10240):
                    if chunk:
                        f.write(chunk)
                f.close()
            t2=time.time()
            print(f'{newList[i]} Download complete \t\t Time   {t2-t1}')

@Gooey(
    program_name='isric Data downloader ',
    encoding="utf-8", )
def main():
    parser = GooeyParser(description="isric Data downloader ")
    parser.add_argument('--url',default=r'https://files.isric.org/soilgrids/latest/data/')
    parser.add_argument('--path', widget="DirChooser", default=r'F:/isricData/')
    args = parser.parse_args()
    url=args.url
    nowPath = args.path
    u = getPath(url)
    rfSearch(u, url,nowPath)
### If visualization is not required, no gooey You can replace the above section with the following 
#@Gooey(
#    program_name='isric Data downloader ',
#   encoding="utf-8", )
# Above 3 You can delete the row 
###main Function with the following section: 
# def main():
#     url=r'https://files.isric.org/soilgrids/latest/data/'# Modify the address link here 
#     nowPath = r'F:/isricData/'# Modify the file save address here 
#     u = getPath(url)
#     rfSearch(u, url,nowPath)

if __name__ == "__main__":
    main()