一些常用的python 小技巧记录

python读取.dat文件

columns = ['Fre', 'Ang', 'Cho','Free', 'Suc', 'Sca']
file_name = 'airfoil_self_noise.dat'
with open(file_name) as f:
    dataset = [[float(item) for item in line.split()] for line in f]
dataset = pd.DataFrame(dataset, columns=columns)

2021/1/6

自动建立文件目录

import os
# 程序自动建立给定的文件目录，存在，跳过，否则，建立这个目录
directory = './machine learning'
if os.path.isdir('%s'%directory):
    pass
else:
    #os.mkdir('%s'%directory)
    os.makedirs('%s'%directory)

pyqt5 建立多线程程序，防止GUI出现“未响应”情况

from PyQt5.QtCore import QRunnable, QThreadPool

# 添加线程
class Worker(QRunnable):
    '''
    Worker thread
    '''
    #  在Worker类中添加构造函数，以便于传入参数
    def __init__(self, allTrainingParams, allTrainingHyperParams):
        super().__init__()
        self.allTrainingParams = allTrainingParams
        self.allTrainingHyperParams = allTrainingHyperParams

    #@pyqtSlot()
    # 在下面添加相关执行时间较长的代码，
    def run(self):
        '''
        Your code goes in this function
        '''
        mlFitting = MLImplementation(self.allTrainingParams, self.allTrainingHyperParams)
        mlFitting.fit()


#在实现中的调用
self.threadpool = QThreadPool()
worker = Worker(allTrainingParams, allTrainingHyperParams)
self.threadpool.start(worker)

reference

2021/1/7

执行长时间的后台程序，添加多线程和进度条

import sys
import time

from PySide2 import QtCore
from PySide2.QtCore import Qt
import PySide2.QtWidgets as QtWidgets


class MainWindow(QtWidgets.QMainWindow):
    """Main window, with one button for exporting stuff"""

    def __init__(self, parent=None):
        super().__init__(parent)
        central_widget = QtWidgets.QWidget(self)
        layout = QtWidgets.QHBoxLayout(self)
        button1 = QtWidgets.QPushButton("Tight operation")
        button2 = QtWidgets.QPushButton("Loose operation")
        button1.clicked.connect(self.export_tight_stuff)
        button2.clicked.connect(self.export_loose_stuff)
        layout.addWidget(button1)
        layout.addWidget(button2)
        central_widget.setLayout(layout)
        self.setCentralWidget(central_widget)

    def export_tight_stuff(self):
        """Opens dialog and starts exporting"""
        worker = Worker(self.tight_operation)
        some_window = MyExportDialog(worker, self)
        some_window.exec_()

    def export_loose_stuff(self):
        """Opens dialog and starts exporting"""

        # self.loose_operation 是传入的要执行的长时间的后台应用程序，
        # 使用自己的应用程序时，可以类比这个程序改写
        worker = Worker(self.loose_operation)

        # QDialog 可以锁定应用程序， 并且能够通过进度条显示程序执行的进度1
        some_window = MyExportDialog(worker, self)
        some_window.exec_()

    @staticmethod
    def loose_operation():
        """Something that doesn't take a lot of CPU power"""
        time.sleep(.1)

    @staticmethod
    def tight_operation():
        """Something that takes a lot of CPU power"""
        some_val = 0
        for i in range(1_000_000):
            some_val += 1


class WorkerSignals(QtCore.QObject):
    progress = QtCore.Signal(int)
    finished = QtCore.Signal()


class Worker(QtCore.QRunnable):

    def __init__(self, fn):
        super().__init__()
        self.operation = fn
        self.signals = WorkerSignals()

    def run(self):
        cnt = 0
        while cnt < 100:
            cnt += 1
            # 添加要执行的后台应用函数
            self.operation()
            self.signals.progress.emit(cnt)
        self.signals.finished.emit()


class MyExportDialog(QtWidgets.QDialog):
    """Dialog which does some stuff, and shows it's progress"""

    def __init__(self, worker, parent=None):
        super().__init__(parent, Qt.WindowCloseButtonHint)
        self.setWindowTitle("Exporting...")
        layout = QtWidgets.QHBoxLayout()
        self.progress_bar = QtWidgets.QProgressBar(self)
        layout.addWidget(self.progress_bar)
        self.setLayout(layout)
        worker.signals.progress.connect(self.progress_bar.setValue)
        worker.signals.finished.connect(self.close)
        self.thread_pool = QtCore.QThreadPool()
        self.thread_pool.start(worker)


if __name__ == "__main__":
    app = QtWidgets.QApplication()
    window = MainWindow()
    window.show()
    sys.exit(app.exec_())

reference

python .png 转化为.ico

# pip install pillow
from PIL import Image


filename = r'earth.png'
img = Image.open(filename)
img.save('earth.ico',format = 'ICO', sizes=[(128,128)])

pandas DataFrame 修改数据小数点位数

import pandas as pd

format = lambda x:'%.2f' % x
df = pd.read_csv('samples.csv')
df = df.applymap(format)

df.to_csv('resamples.csv')

遍历给定数目的天数

from datetime import date, timedelta

start_date = date(2010, 1, 1)
end_date = date(2010, 12, 27)
delta = timedelta(days=8)
while start_date <= end_date:
    print (start_date.strftime("%Y.%m.%d"))
    start_date += delta

添加遍历数字

["%.2d" % i for i in range(1, 25)]
>>>['01',
 '02',
 '03',
 '04',
 '05',
 '06',
 '07',
 '08',
 '09',
 '10',
 '11',
 '12',
 '13',
 '14',
 '15',
 '16',
 '17',
 '18',
 '19',
 '20',
 '21',
 '22',
 '23',
 '24']

## applying function for pandas
f = lambda x:x**2

data = {'a':[1,2,3,4], 'b':[5,6,7,8]}
df = pd.DataFrame(data)
print(df)
df.apply(f)
df.applymap(f)

Pandas select

# 使用groupby.filter() 过滤数据
a = loc_data.groupby(["row","col"]).filter(lambda x: len(x) == 2)
print(a.shape)
a.to_csv('222.csv')

shutil copy file

file_lists = os.listdir(file_dir)
file_lists = [file for file in file_lists if file.endswith('.hdf')]
for file in file_lists:
     if file[20:23] in v_lists and file[17:20] in h_lists:
     shutil.copy(file_dir + '/%s' % file, file_dir + '/region/%s' % file)
   

commonly used regression metrics

from sklearn.metrics import mean_squared_error, r2_score, mean_absolute_error, mean_absolute_percentage_error
import numpy as np 

def rmse(y_true, y_pred):
    
    return np.sqrt(mean_squared_error(y_true, y_pred))

def rrmse(y_true, y_pred):
    
    return rmse(y_true, y_pred) / np.mean(y_true)

def bias(y_true, y_pred):
    return np.mean(y_pred) - np.mean(y_true)


metrics_dict = {'R2':r2_score,
                'RMSE':rmse,
                'bias':bias,
                 'rRMSE':rrmse,
                 'MAE':mean_absolute_error,
                 'MAPE':mean_absolute_percentage_error}