\n\nint main(){\n int arr [10] = {2,3,4,5,6,7,8,9,10,11};\n arr[0] = 1525;\n *(arr+1) = 25;\n int len = sizeof(arr)/sizeof(arr[0]);\n void printArr(int arr[], int len){\n for (int i = 0; i < len;i++){\n printf(\"%d\\t\",arr[i]);\n }\n }\n printArr(arr,len);\n return 0;\n}\n```\n### 指针\n```c\n // swap the value of a and b\n void swap(int* x, int* y){\n int temp = *x;\n *x = *y;\n *y = temp;\n\n }\n int a = 5;\n int b = 10;\n swap(&a, &b);\n printf(\"a = %d b = %d\\n\", a, b);\n```\n","slug":"language/C","published":1,"updated":"2025-01-24T10:16:05.649Z","comments":1,"layout":"post","photos":[],"_id":"cmgsv0jne00093wahhwfe0rze","content":"c lang在windows下的开发（VS code） WinLibs - GCC+MinGW-w64 compiler for Windows 下载你需要的版本D:\\ProgramModule，并将 bin\\加入环境变量PATHTerminal输入gcc -v，查看gcc是否安装成功VS code 的插件管理下载Code Runner、C\\C++这两个插件*.c源文件的内容区，右键点击Run Code ，即可运行成功
\n数据类型 \n整数类型1 2 3 4 5 6 7 8 9 10 11 short a = 12 ; int b = 100 ; long c = 1000L ; long long d = 1000000LL ; unsigned int e = 10 ; printf ("a: %hd\\n" ,a); printf ("b: %d\\n" ,b); printf ("c: %ld\\n" ,c); printf ("d: %lld\\n" ,d); printf ("e: %u\\n" ,e); printf ("f: %.3f\\n" ,f);
\n小数类型1 2 3 4 float f = 3.14F ;printf ("f: %.3f\\n" ,f);double g = 5.65 ;printf ("g: %.2lf\\n" ,g);
\n字符类型1 2 char h = 'x' ;printf ("x: %c\\n" ,x);
\n \n类型转换 \n隐式转换 \n强制转换1 2 int b = 23 ;short c = (short ) b;
\n \n数组 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 #include <stdio.h> int main () { int arr [10 ] = {2 ,3 ,4 ,5 ,6 ,7 ,8 ,9 ,10 ,11 }; arr[0 ] = 1525 ; *(arr+1 ) = 25 ; int len = sizeof (arr)/sizeof (arr[0 ]); void printArr (int arr[], int len) { for (int i = 0 ; i < len;i++){ printf ("%d\\t" ,arr[i]); } } printArr(arr,len); return 0 ; }
指针 1 2 3 4 5 6 7 8 9 10 11 void swap (int * x, int * y) { int temp = *x; *x = *y; *y = temp; } int a = 5 ; int b = 10 ; swap(&a, &b); printf ("a = %d b = %d\\n" , a, b);
c lang在windows下的开发（VS code） WinLibs - GCC+MinGW-w64 compiler for Windows 下载你需要的版本D:\\ProgramModule，并将 bin\\加入环境变量PATHTerminal输入gcc -v，查看gcc是否安装成功VS code 的插件管理下载Code Runner、C\\C++这两个插件*.c源文件的内容区，右键点击Run Code ，即可运行成功
\n数据类型 \n整数类型1 2 3 4 5 6 7 8 9 10 11 short a = 12 ; int b = 100 ; long c = 1000L ; long long d = 1000000LL ; unsigned int e = 10 ; printf ("a: %hd\\n" ,a); printf ("b: %d\\n" ,b); printf ("c: %ld\\n" ,c); printf ("d: %lld\\n" ,d); printf ("e: %u\\n" ,e); printf ("f: %.3f\\n" ,f);
\n小数类型1 2 3 4 float f = 3.14F ;printf ("f: %.3f\\n" ,f);double g = 5.65 ;printf ("g: %.2lf\\n" ,g);
\n字符类型1 2 char h = 'x' ;printf ("x: %c\\n" ,x);
\n \n类型转换 \n隐式转换 \n强制转换1 2 int b = 23 ;short c = (short ) b;
\n \n数组 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 #include <stdio.h> int main () { int arr [10 ] = {2 ,3 ,4 ,5 ,6 ,7 ,8 ,9 ,10 ,11 }; arr[0 ] = 1525 ; *(arr+1 ) = 25 ; int len = sizeof (arr)/sizeof (arr[0 ]); void printArr (int arr[], int len) { for (int i = 0 ; i < len;i++){ printf ("%d\\t" ,arr[i]); } } printArr(arr,len); return 0 ; }
指针 1 2 3 4 5 6 7 8 9 10 11 void swap (int * x, int * y) { int temp = *x; *x = *y; *y = temp; } int a = 5 ; int b = 10 ; swap(&a, &b); printf ("a = %d b = %d\\n" , a, b);
C4.5 C4.5 是一种用于生成决策树的算法，不再使用信息增益，而是使用信息增益比，来避免偏向于选择取值较多的特征。信息增益比是信息增益与特征的熵的比值。
\nID3 D3 是一种基于信息增益（Information Gain）的决策树算法
\nCart CART（分类与回归树）一种决策树算法，，CART 使用 二叉树结构 ，即每个节点只能有两个子节点。
\ncart剪枝 CART 决策树的剪枝方法分为 预剪枝 （Pre-pruning）和 后剪枝 （Post-pruning）两种：
\n预剪枝 ：预剪枝是在构建决策树时就决定是否停止进一步划分某个节点。主要通过以下标准来控制：
\n\n当某个节点的样本数小于某个阈值时，不再继续划分。 \n当某个节点的 Gini 不纯度小于某个阈值时，不再继续划分。 \n \n后剪枝 ：后剪枝是在决策树完全构建出来之后，对树进行修剪。具体过程如下：
\n\n构建完整的决策树。 \n从叶子节点开始，逐渐向上遍历树的每个节点。 \n对每个节点进行判断，是否合适剪去该节点及其子树。如果剪去该子树后，模型的性能没有显著下降，就可以剪枝。 \n \n特征工程(特征提取) \n字典特征提取 1 2 3 4 5 6 7 8 9 10 11 from sklearn.feature_extraction import DictVectorizer data = [ {'city' : 'beijing' , 'temperature' : 100 }, {'city' : 'shanghai' , 'temperature' : 95 }, {'city' : 'guangzhou' , 'temperature' : 98 } ] transfer = DictVectorizer(sparse=False ) new_data = transfer.fit_transform(data) print (transfer.feature_names_) print (new_data)
文本特征提取 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 from sklearn.feature_extraction.text import CountVectorizer data = [ "I love programming" , "Python is great" , "I love machine learning" ] transfer = CountVectorizer() new_data = transfer.fit_transform(data) print ("Feature Names:" , transfer.get_feature_names_out()) print ("Transformed Data:" , new_data.toarray())
文本特征提取（中文文本) 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 from sklearn.feature_extraction.text import CountVectorizer import jieba data = [ "我热爱编程，学习编程语言是一件非常有趣的事情，它能够提升我们解决问题的能力，编程让我变得更加有创意。" , "Python语言是一门非常强大的编程语言，具有简洁的语法和丰富的库，可以帮助开发者更高效地完成任务。" , "机器学习是一项非常有前途的技术，它能够让计算机从数据中自动学习，逐步提高模型的精确度，解决实际问题。" ] text_list = [] for line in data: text_list.append(" " .join(list (jieba.cut(line)))) transfer = CountVectorizer() new_data = transfer.fit_transform(text_list) print ("Feature Names:" , transfer.get_feature_names_out()) print ("Transformed Data:" , new_data.toarray())
\n \ntf-idf \n词频 * 逆文档频率
\n \n1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 from sklearn.feature_extraction.text import TfidfVectorizerimport jiebadata=["一种还是一种今天很残酷，明天更残酷，后天很美好，但绝对大部分是死在明天晚上，所以每个人不要放弃今天。" , "我们看到的从很远星系来的光是在几百万年之前发出的，这样当我们看到宇宙时，我们是在看它的过去" , "如果只用一种方式了解某样事物，你就不会真正了解它。了解事物真正含义的秘密取决于如何将其与我们所了解的事物相联系" ] list = []for item in data: list .append(" " .join(jieba.cut(item))) transfer = TfidfVectorizer() new_data = transfer.fit_transform(list ) print (f"特征名字：\\n{transfer.get_feature_names_out()} " )print (f"转换后的特征矩阵:\\n{ new_data.toarray()} " )print (f"转换后的数据:\\n{new_data} " )
回归决策树 决策树算法的应用 （泰坦尼克号沉船幸存者预测） 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 import seaborn as snsimport pandas as pdimport numpy as npfrom sklearn.model_selection import train_test_splitfrom sklearn.feature_extraction import DictVectorizerfrom sklearn.tree import DecisionTreeClassifier,export_graphviztitanic = sns.load_dataset('titanic' ) missing_age_count = titanic['age' ].isna().sum () X = titanic[['pclass' ,'age' ,'sex' ]] y = titanic['survived' ] X.loc[:, 'age' ] = X['age' ].fillna(value=X['age' ].mean()) X_train,X_test,y_train,y_test = train_test_split(X,y,random_state=22 ) X_train = X_train.to_dict(orient="records" ) X_test= X_test.to_dict(orient="records" ) transfer = DictVectorizer() X_train = transfer.fit_transform(X_train) X_test = transfer.transform(X_test) estimator = DecisionTreeClassifier(criterion="gini" ) estimator.fit(X_train,y_train) y_pred = estimator.predict(X_test) print (f"模型的测试集的预测值：{y_pred} " )ret = estimator.score(X_test,y_test) print (f"模型的评分:{ret} " )print (X_test.toarray())
生成对应的图
\n1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 from sklearn.tree import export_graphvizimport graphviz export_graphviz(estimator, out_file='./data/tree.dot' , feature_names=transfer.get_feature_names_out()) with open ('./data/tree.dot' , 'r' ) as f: dot_graph = f.read() graph = graphviz.Source(dot_graph) output_path = './data/decision_tree' graph.view(output_path)
Webgraphviz ，这个网站可以将tree.dot文件的内容生成对应的可视化树
\n回归决策树与线性回归的对比 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 import numpy as npimport matplotlib.pyplot as pltfrom sklearn.linear_model import LinearRegressionfrom sklearn.tree import DecisionTreeRegressorfrom matplotlib import rcParamsrcParams['font.sans-serif' ] = ['SimHei' ] rcParams['axes.unicode_minus' ] = False x = np.array(list (range (1 ,11 ))).reshape(-1 ,1 ) y = ([5.56 ,5.70 ,5.91 ,6.40 ,6.80 ,7.05 ,8.90 ,8.70 ,9.00 ,9.05 ]) m1 = DecisionTreeRegressor(max_depth=1 ) m2 = DecisionTreeRegressor(max_depth=3 ) m3 = DecisionTreeRegressor() m1.fit(x,y) m2.fit(x,y) m3.fit(x,y) x_test = np.arange(0 ,10 ,0.01 ).reshape(-1 ,1 ) y_1 = m1.predict(x_test) y_2 = m2.predict(x_test) y_3 = m3.predict(x_test) plt.figure(figsize=(10 ,6 ),dpi=100 ) plt.scatter(x,y ,label = "data" ) plt.plot(x_test,y_1,label = "max_depth=1" ) plt.plot(x_test,y_2,label = "max_depth=3" ) plt.plot(x_test,y_3,label = "linearregression" ) plt.xlabel("数据" ) plt.ylabel("预测值" ) plt.legend() plt.show()
C4.5 C4.5 是一种用于生成决策树的算法，不再使用信息增益，而是使用信息增益比，来避免偏向于选择取值较多的特征。信息增益比是信息增益与特征的熵的比值。
\nID3 D3 是一种基于信息增益（Information Gain）的决策树算法
\nCart CART（分类与回归树）一种决策树算法，，CART 使用 二叉树结构 ，即每个节点只能有两个子节点。
\ncart剪枝 CART 决策树的剪枝方法分为 预剪枝 （Pre-pruning）和 后剪枝 （Post-pruning）两种：
\n预剪枝 ：预剪枝是在构建决策树时就决定是否停止进一步划分某个节点。主要通过以下标准来控制：
\n\n当某个节点的样本数小于某个阈值时，不再继续划分。 \n当某个节点的 Gini 不纯度小于某个阈值时，不再继续划分。 \n \n后剪枝 ：后剪枝是在决策树完全构建出来之后，对树进行修剪。具体过程如下：
\n\n构建完整的决策树。 \n从叶子节点开始，逐渐向上遍历树的每个节点。 \n对每个节点进行判断，是否合适剪去该节点及其子树。如果剪去该子树后，模型的性能没有显著下降，就可以剪枝。 \n \n特征工程(特征提取) \n字典特征提取 1 2 3 4 5 6 7 8 9 10 11 from sklearn.feature_extraction import DictVectorizer data = [ {'city' : 'beijing' , 'temperature' : 100 }, {'city' : 'shanghai' , 'temperature' : 95 }, {'city' : 'guangzhou' , 'temperature' : 98 } ] transfer = DictVectorizer(sparse=False ) new_data = transfer.fit_transform(data) print (transfer.feature_names_) print (new_data)
文本特征提取 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 from sklearn.feature_extraction.text import CountVectorizer data = [ "I love programming" , "Python is great" , "I love machine learning" ] transfer = CountVectorizer() new_data = transfer.fit_transform(data) print ("Feature Names:" , transfer.get_feature_names_out()) print ("Transformed Data:" , new_data.toarray())
文本特征提取（中文文本) 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 from sklearn.feature_extraction.text import CountVectorizer import jieba data = [ "我热爱编程，学习编程语言是一件非常有趣的事情，它能够提升我们解决问题的能力，编程让我变得更加有创意。" , "Python语言是一门非常强大的编程语言，具有简洁的语法和丰富的库，可以帮助开发者更高效地完成任务。" , "机器学习是一项非常有前途的技术，它能够让计算机从数据中自动学习，逐步提高模型的精确度，解决实际问题。" ] text_list = [] for line in data: text_list.append(" " .join(list (jieba.cut(line)))) transfer = CountVectorizer() new_data = transfer.fit_transform(text_list) print ("Feature Names:" , transfer.get_feature_names_out()) print ("Transformed Data:" , new_data.toarray())
\n \ntf-idf \n词频 * 逆文档频率
\n \n1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 from sklearn.feature_extraction.text import TfidfVectorizerimport jiebadata=["一种还是一种今天很残酷，明天更残酷，后天很美好，但绝对大部分是死在明天晚上，所以每个人不要放弃今天。" , "我们看到的从很远星系来的光是在几百万年之前发出的，这样当我们看到宇宙时，我们是在看它的过去" , "如果只用一种方式了解某样事物，你就不会真正了解它。了解事物真正含义的秘密取决于如何将其与我们所了解的事物相联系" ] list = []for item in data: list .append(" " .join(jieba.cut(item))) transfer = TfidfVectorizer() new_data = transfer.fit_transform(list ) print (f"特征名字：\\n{transfer.get_feature_names_out()} " )print (f"转换后的特征矩阵:\\n{ new_data.toarray()} " )print (f"转换后的数据:\\n{new_data} " )
回归决策树 决策树算法的应用 （泰坦尼克号沉船幸存者预测） 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 import seaborn as snsimport pandas as pdimport numpy as npfrom sklearn.model_selection import train_test_splitfrom sklearn.feature_extraction import DictVectorizerfrom sklearn.tree import DecisionTreeClassifier,export_graphviztitanic = sns.load_dataset('titanic' ) missing_age_count = titanic['age' ].isna().sum () X = titanic[['pclass' ,'age' ,'sex' ]] y = titanic['survived' ] X.loc[:, 'age' ] = X['age' ].fillna(value=X['age' ].mean()) X_train,X_test,y_train,y_test = train_test_split(X,y,random_state=22 ) X_train = X_train.to_dict(orient="records" ) X_test= X_test.to_dict(orient="records" ) transfer = DictVectorizer() X_train = transfer.fit_transform(X_train) X_test = transfer.transform(X_test) estimator = DecisionTreeClassifier(criterion="gini" ) estimator.fit(X_train,y_train) y_pred = estimator.predict(X_test) print (f"模型的测试集的预测值：{y_pred} " )ret = estimator.score(X_test,y_test) print (f"模型的评分:{ret} " )print (X_test.toarray())
生成对应的图
\n1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 from sklearn.tree import export_graphvizimport graphviz export_graphviz(estimator, out_file='./data/tree.dot' , feature_names=transfer.get_feature_names_out()) with open ('./data/tree.dot' , 'r' ) as f: dot_graph = f.read() graph = graphviz.Source(dot_graph) output_path = './data/decision_tree' graph.view(output_path)
Webgraphviz ，这个网站可以将tree.dot文件的内容生成对应的可视化树
\n回归决策树与线性回归的对比 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 import numpy as npimport matplotlib.pyplot as pltfrom sklearn.linear_model import LinearRegressionfrom sklearn.tree import DecisionTreeRegressorfrom matplotlib import rcParamsrcParams['font.sans-serif' ] = ['SimHei' ] rcParams['axes.unicode_minus' ] = False x = np.array(list (range (1 ,11 ))).reshape(-1 ,1 ) y = ([5.56 ,5.70 ,5.91 ,6.40 ,6.80 ,7.05 ,8.90 ,8.70 ,9.00 ,9.05 ]) m1 = DecisionTreeRegressor(max_depth=1 ) m2 = DecisionTreeRegressor(max_depth=3 ) m3 = DecisionTreeRegressor() m1.fit(x,y) m2.fit(x,y) m3.fit(x,y) x_test = np.arange(0 ,10 ,0.01 ).reshape(-1 ,1 ) y_1 = m1.predict(x_test) y_2 = m2.predict(x_test) y_3 = m3.predict(x_test) plt.figure(figsize=(10 ,6 ),dpi=100 ) plt.scatter(x,y ,label = "data" ) plt.plot(x_test,y_1,label = "max_depth=1" ) plt.plot(x_test,y_2,label = "max_depth=3" ) plt.plot(x_test,y_3,label = "linearregression" ) plt.xlabel("数据" ) plt.ylabel("预测值" ) plt.legend() plt.show()
Bagging 随机森林 \nRandom-Forest 就是Bagging + Decisiontree
\n \n1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 import seaborn as snsimport pandas as pdimport numpy as npfrom sklearn.model_selection import train_test_split,GridSearchCVfrom sklearn.feature_extraction import DictVectorizerfrom sklearn.ensemble import RandomForestClassifiertitanic = sns.load_dataset('titanic' ) missing_age_count = titanic['age' ].isna().sum () X = titanic[['pclass' ,'age' ,'sex' ]] y = titanic['survived' ] X.loc[:, 'age' ] = X['age' ].fillna(value=X['age' ].mean()) X_train,X_test,y_train,y_test = train_test_split(X,y,random_state=22 ) X_train = X_train.to_dict(orient="records" ) X_test= X_test.to_dict(orient="records" ) transfer = DictVectorizer() X_train = transfer.fit_transform(X_train) X_test = transfer.transform(X_test) rf = RandomForestClassifier() gc = GridSearchCV(estimator=rf ,param_grid={"n_estimators" :[100 ,120 ,300 ],"max_depth" :[3 ,7 ,11 ]},cv=3 ) gc.fit(X_train,y_train) y_pred = gc.predict(X_test) print (f"模型的测试集的预测值：{y_pred} " )ret = gc.score(X_test,y_test) print (f"最佳模型在测试集上的评分:{ret} " )print (f"最佳模型的参数：{gc.best_estimator_} " )print (f"最佳模型在训练集上的评分：{gc.best_score_} " )print (X_test.toarray())
\nott案例 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 import pandas as pdimport numpy as npimport matplotlib.pyplot as pltfrom imblearn.under_sampling import RandomUnderSamplerfrom sklearn.model_selection import train_test_splitfrom sklearn.preprocessing import LabelEncoderfrom sklearn.ensemble import RandomForestClassifierfrom sklearn.metrics import log_lossfrom sklearn.preprocessing import OneHotEncoderdata = pd.read_csv('./data/train.csv' ) import seaborn as snssns.countplot(data=data, x='target' , hue='target' , palette="Set2" , legend=False ) plt.show() x = data.drop(["id" , "target" ], axis=1 ) y = data['target' ] undersampler = RandomUnderSampler(sampling_strategy='auto' , random_state=0 ) x_resampled, y_resampled = undersampler.fit_resample(x, y) le = LabelEncoder() y_resampled = le.fit_transform(y_resampled) x_train, x_test, y_train, y_test = train_test_split(x_resampled, y_resampled, test_size=0.2 ) rf = RandomForestClassifier(oob_score = True ) rf.fit(x_train,y_train) y_pred = rf.predict(x_test) print (f"预测值:{y_pred} " )print (f"评分：{rf.score(x_test,y_test)} " )y_pred_prob = rf.predict_proba(x_test) encoder = OneHotEncoder(sparse_output=False ) y_test_one_hot = encoder.fit_transform(y_test.reshape(-1 , 1 )) eps = 1e-15 y_pred_prob = np.clip(y_pred_prob, eps, 1 - eps) loss = log_loss(y_test_one_hot, y_pred_prob, normalize=True ) print (f"Log Loss: {loss} " )
\n","cover_type":"img","excerpt":"","more":"Bagging 随机森林 \nRandom-Forest 就是Bagging + Decisiontree
\n \n1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 import seaborn as snsimport pandas as pdimport numpy as npfrom sklearn.model_selection import train_test_split,GridSearchCVfrom sklearn.feature_extraction import DictVectorizerfrom sklearn.ensemble import RandomForestClassifiertitanic = sns.load_dataset('titanic' ) missing_age_count = titanic['age' ].isna().sum () X = titanic[['pclass' ,'age' ,'sex' ]] y = titanic['survived' ] X.loc[:, 'age' ] = X['age' ].fillna(value=X['age' ].mean()) X_train,X_test,y_train,y_test = train_test_split(X,y,random_state=22 ) X_train = X_train.to_dict(orient="records" ) X_test= X_test.to_dict(orient="records" ) transfer = DictVectorizer() X_train = transfer.fit_transform(X_train) X_test = transfer.transform(X_test) rf = RandomForestClassifier() gc = GridSearchCV(estimator=rf ,param_grid={"n_estimators" :[100 ,120 ,300 ],"max_depth" :[3 ,7 ,11 ]},cv=3 ) gc.fit(X_train,y_train) y_pred = gc.predict(X_test) print (f"模型的测试集的预测值：{y_pred} " )ret = gc.score(X_test,y_test) print (f"最佳模型在测试集上的评分:{ret} " )print (f"最佳模型的参数：{gc.best_estimator_} " )print (f"最佳模型在训练集上的评分：{gc.best_score_} " )print (X_test.toarray())
\nott案例 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 import pandas as pdimport numpy as npimport matplotlib.pyplot as pltfrom imblearn.under_sampling import RandomUnderSamplerfrom sklearn.model_selection import train_test_splitfrom sklearn.preprocessing import LabelEncoderfrom sklearn.ensemble import RandomForestClassifierfrom sklearn.metrics import log_lossfrom sklearn.preprocessing import OneHotEncoderdata = pd.read_csv('./data/train.csv' ) import seaborn as snssns.countplot(data=data, x='target' , hue='target' , palette="Set2" , legend=False ) plt.show() x = data.drop(["id" , "target" ], axis=1 ) y = data['target' ] undersampler = RandomUnderSampler(sampling_strategy='auto' , random_state=0 ) x_resampled, y_resampled = undersampler.fit_resample(x, y) le = LabelEncoder() y_resampled = le.fit_transform(y_resampled) x_train, x_test, y_train, y_test = train_test_split(x_resampled, y_resampled, test_size=0.2 ) rf = RandomForestClassifier(oob_score = True ) rf.fit(x_train,y_train) y_pred = rf.predict(x_test) print (f"预测值:{y_pred} " )print (f"评分：{rf.score(x_test,y_test)} " )y_pred_prob = rf.predict_proba(x_test) encoder = OneHotEncoder(sparse_output=False ) y_test_one_hot = encoder.fit_transform(y_test.reshape(-1 , 1 )) eps = 1e-15 y_pred_prob = np.clip(y_pred_prob, eps, 1 - eps) loss = log_loss(y_test_one_hot, y_pred_prob, normalize=True ) print (f"Log Loss: {loss} " )
\n"},{"title":"k近邻算法（K-Nearest Neighbors）KNN","abbrlink":29139,"mathjax":true,"date":"2025-01-13T09:20:59.000Z","_content":"## **k近邻算法（K-Nearest Neighbors）KNN**\n将当前样本的类别归类于距离最近的**k**个样本的类别\n\n#### **距离公式(2维)**\n\n- 欧式距离\n$$\nd = \\sqrt{(x_1-y_1)^2 + (x_2 - y_2)^2}\n$$\n- 曼哈顿距离\n$$\nd = |x_1 - x_2| + |y_1 - y_2|\n$$\n- 切比雪夫距离\n$$\nd = \\max\\left(|x_1 - x_2|, |y_1 - y_2|\\right)\n$$\n#### k值选择问题\n\n| k值 | 影响 |\n| --- | ------------------ |\n| 越大 | 模型过拟合，准确率波动较大 |\n| 越小 | 模型欠拟合，准确率趋于稳定但可能较低 |\n### 特征预处理\n> 通过一些转换函数将特征数据转换成更加适合算法模型的特征数据过程 \n- 归一化\n 将数据变换到指定区间（默认是\\[0,1\\]）\n $$ x' = \\frac{x- x_{\\text {min}}}{x_{\\text{max}} - x_{\\text{min}}} $$\n 若需要缩放到任意区间 \\(\\[a, b\\]\\)，公式为： $$ x' = a + \\frac{(x - x_{\\text{min}}) \\cdot (b - a)}{x_{\\text{max}} - x_{\\text{min}}} $$\n 其中：\\( \\[a, b\\] \\)：目标区间的范围\n 归一化受到数据集的异常值的影响，需要进行标准化处理（更加合理）\n ``` python\n\t from sklearn.preprocessing import MinMaxScaler # 归一化\n\t```\n- 标准化\n 将数据调整为均值为 0，标准差为 1 的标准正态分布\n $$ z = \\frac{x - \\mu}{\\sigma} $$\n \\( z \\)：标准化后的值 \\( x \\)：原始数据值 \\( $\\mu$ \\)：数据的均值 \\( $\\sigma$\\)：数据的标准差\n \n ``` python\n \t from sklearn.preprocessing import StandardScaler # 标准化\n ```\n\n### KNN代码实现\n```python\nimport seaborn as sns\nimport matplotlib.pyplot as plt \nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.datasets import load_iris\nfrom sklearn.preprocessing import MinMaxScaler,StandardScaler\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.metrics import accuracy_score\n\n# 1 数据集获取\niris = load_iris()\n# print(iris.feature_names)\niris_data = pd.DataFrame(iris.data,columns=['Sepal_Length', 'Sepal_Width', 'Petal_Length', 'Petal_Width'])\niris_data['target'] = iris.target\n\ndef iris_plot(data,col1,col2):\n sns.lmplot(x=col1,y=col2,data=data,hue=\"target\",fit_reg=False)\n plt.show()\n# 2 数据集可视化\n# iris_plot(iris_data, 'Sepal_Width', 'Petal_Length')\n\n# 3 数据集的划分\nX_train,X_test,y_train,y_test = train_test_split(iris.data,iris.target,test_size=0.2,random_state=44)\n# print(\"训练集的特征值：\\n\",X_train)\n# print(\"训练集的目标值：\\n\",y_train)\n# print(\"测试集的特征值：\\n\",X_test)\n# print(\"测试集的特征值：\\n\",y_test)\n\n# 4 归一化\ntransfer = StandardScaler()\nX_train = transfer.fit_transform(X_train)\nX_test = transfer.transform(X_test)\n# print(\"归一化的,X_train：\\n\",X_train)\n# print(\"归一化的X_test：\\n\",X_test)\n\n# 5 机器学习 KNN\n# 5.1 实例化估计器\nestimator = KNeighborsClassifier(n_neighbors=9)\n# 5.2 进行训练\nestimator.fit(X_train,y_train)\n\n# 6 模型评估\ny_pred = estimator.predict(X_test)\nprint(\"预测值：\\n\",y_pre)\nprint(\"预测值与真实值是否相等：\\n\",y_pred==y_test)\naccuracy = accuracy_score(y_test, y_pred)\nprint(f\"\\nKNN 模型的准确率: {accuracy:.4f}\")\n```\n\n\n### 交叉验证与网格搜索\n```python\nimport seaborn as sns\nimport matplotlib.pyplot as plt \nimport pandas as pd\nfrom sklearn.model_selection import train_test_split,GridSearchCV\nfrom sklearn.datasets import load_iris\nfrom sklearn.preprocessing import MinMaxScaler,StandardScaler\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.metrics import accuracy_score\n\n# 1 数据集获取\niris = load_iris()\niris_data = pd.DataFrame(iris.data,columns=['Sepal_Length', 'Sepal_Width', 'Petal_Length', 'Petal_Width'])\niris_data['target'] = iris.target\n\n# 3 数据集的划分\nX_train,X_test,y_train,y_test = train_test_split(iris.data,iris.target,test_size=0.2)\n\n# 4 归一化\ntransfer = StandardScaler()\nX_train = transfer.fit_transform(X_train)\nX_test = transfer.transform(X_test)\n\n# 5 机器学习 KNN\n# 5.1 实例化估计器\n#\n#不指定 n_neighbors ,使用网格搜索进行循环训练\nestimator = KNeighborsClassifier()\n# 5.2 模型调优 -- 交叉验证，网格搜素\nestimator = GridSearchCV(estimator,param_grid={\"n_neighbors\":[1,3,5,7]},cv=5) # 5 折\n# 5.2 进行训练\nestimator.fit(X_train,y_train)\n \n# 6 模型评估\ny_pred = estimator.predict(X_test)\nprint(\"预测值：\\n\",y_pred)\nprint(\"预测值与真实值是否相等：\\n\",y_pred==y_test)\naccuracy = accuracy_score(y_test, y_pred)\nprint(f\"\\nKNN 模型的准确率: {accuracy:.4f}\")\n\n# 交叉验证的相关参数\nprint(f\"最好结果：{estimator.best_score_}\")\nprint(f\"最好模型：{estimator.best_estimator_}\")\nprint(f\"最好模型结果：{estimator.cv_results_}\")\n```\n\n\n### 机器学习的基本步骤\n- 获取数据集\n- 数据集基本处理\n - 去重去空、填充等操作 \n - 确定特征值和目标值\n - 分割数据集\n- 特征工程（特征预处理 标准化等）\n- 机器学习\n- 模型评估\n\n### 数据分割的方法\n- 留出法\n 训练/测试集的划分要尽可能保持数据分布的一致性，避免因数据划分过程引入额外的偏差而对最终结果产生影响。\n 单次使用留出法得到的估计结果往往不够稳定可靠，在使用留出法时，一般要采用若干次随机划分、重复进行实验评估后取平均值作为留出法的评估结果。\n ``` python\n\tfrom sklearn.model_selection import KFold,StratifiedKFold\n\timport pandas as pd\n\tX = np.array([\n\t[1,2,3,4],\n\t[11,12,13,14],\n\t[21,22,23,24],\n\t[31,32,33,34],\n\t[41,42,43,44],\n\t[51,52,53,54],\n\t[61,62,63,64],\n\t[71,72,73,74]\n\t])\n\ty=np.array([1,1,0,0,1,1,0,0])\n\tfolder = KFold(n_splits=4)\n\tsfloder = StratifiedKFold(n_splits=4)\n\tprint(\"KFOLD:\")\n\tfor train,test in folder.split(X,y):\n\t print(f\"train:{train},test:{test}\")\n\tprint(\"SKFOLD:\")\n\tfor train,test in sfloder.split(X,y):\n print(f\"train:{train},test:{test}\")\n\t```\n\t\n- 自助法\n- 交叉验证法","source":"_posts/machinelearning/knn.md","raw":"---\ntitle: k近邻算法（K-Nearest Neighbors）KNN\ntags: KNN\ncategories: machinelearning\nabbrlink: 29139\nmathjax: true\ndate: 2025-01-13 17:20:59\n---\n## **k近邻算法（K-Nearest Neighbors）KNN**\n将当前样本的类别归类于距离最近的**k**个样本的类别\n\n#### **距离公式(2维)**\n\n- 欧式距离\n$$\nd = \\sqrt{(x_1-y_1)^2 + (x_2 - y_2)^2}\n$$\n- 曼哈顿距离\n$$\nd = |x_1 - x_2| + |y_1 - y_2|\n$$\n- 切比雪夫距离\n$$\nd = \\max\\left(|x_1 - x_2|, |y_1 - y_2|\\right)\n$$\n#### k值选择问题\n\n| k值 | 影响 |\n| --- | ------------------ |\n| 越大 | 模型过拟合，准确率波动较大 |\n| 越小 | 模型欠拟合，准确率趋于稳定但可能较低 |\n### 特征预处理\n> 通过一些转换函数将特征数据转换成更加适合算法模型的特征数据过程 \n- 归一化\n 将数据变换到指定区间（默认是\\[0,1\\]）\n $$ x' = \\frac{x- x_{\\text {min}}}{x_{\\text{max}} - x_{\\text{min}}} $$\n 若需要缩放到任意区间 \\(\\[a, b\\]\\)，公式为： $$ x' = a + \\frac{(x - x_{\\text{min}}) \\cdot (b - a)}{x_{\\text{max}} - x_{\\text{min}}} $$\n 其中：\\( \\[a, b\\] \\)：目标区间的范围\n 归一化受到数据集的异常值的影响，需要进行标准化处理（更加合理）\n ``` python\n\t from sklearn.preprocessing import MinMaxScaler # 归一化\n\t```\n- 标准化\n 将数据调整为均值为 0，标准差为 1 的标准正态分布\n $$ z = \\frac{x - \\mu}{\\sigma} $$\n \\( z \\)：标准化后的值 \\( x \\)：原始数据值 \\( $\\mu$ \\)：数据的均值 \\( $\\sigma$\\)：数据的标准差\n \n ``` python\n \t from sklearn.preprocessing import StandardScaler # 标准化\n ```\n\n### KNN代码实现\n```python\nimport seaborn as sns\nimport matplotlib.pyplot as plt \nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.datasets import load_iris\nfrom sklearn.preprocessing import MinMaxScaler,StandardScaler\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.metrics import accuracy_score\n\n# 1 数据集获取\niris = load_iris()\n# print(iris.feature_names)\niris_data = pd.DataFrame(iris.data,columns=['Sepal_Length', 'Sepal_Width', 'Petal_Length', 'Petal_Width'])\niris_data['target'] = iris.target\n\ndef iris_plot(data,col1,col2):\n sns.lmplot(x=col1,y=col2,data=data,hue=\"target\",fit_reg=False)\n plt.show()\n# 2 数据集可视化\n# iris_plot(iris_data, 'Sepal_Width', 'Petal_Length')\n\n# 3 数据集的划分\nX_train,X_test,y_train,y_test = train_test_split(iris.data,iris.target,test_size=0.2,random_state=44)\n# print(\"训练集的特征值：\\n\",X_train)\n# print(\"训练集的目标值：\\n\",y_train)\n# print(\"测试集的特征值：\\n\",X_test)\n# print(\"测试集的特征值：\\n\",y_test)\n\n# 4 归一化\ntransfer = StandardScaler()\nX_train = transfer.fit_transform(X_train)\nX_test = transfer.transform(X_test)\n# print(\"归一化的,X_train：\\n\",X_train)\n# print(\"归一化的X_test：\\n\",X_test)\n\n# 5 机器学习 KNN\n# 5.1 实例化估计器\nestimator = KNeighborsClassifier(n_neighbors=9)\n# 5.2 进行训练\nestimator.fit(X_train,y_train)\n\n# 6 模型评估\ny_pred = estimator.predict(X_test)\nprint(\"预测值：\\n\",y_pre)\nprint(\"预测值与真实值是否相等：\\n\",y_pred==y_test)\naccuracy = accuracy_score(y_test, y_pred)\nprint(f\"\\nKNN 模型的准确率: {accuracy:.4f}\")\n```\n\n\n### 交叉验证与网格搜索\n```python\nimport seaborn as sns\nimport matplotlib.pyplot as plt \nimport pandas as pd\nfrom sklearn.model_selection import train_test_split,GridSearchCV\nfrom sklearn.datasets import load_iris\nfrom sklearn.preprocessing import MinMaxScaler,StandardScaler\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.metrics import accuracy_score\n\n# 1 数据集获取\niris = load_iris()\niris_data = pd.DataFrame(iris.data,columns=['Sepal_Length', 'Sepal_Width', 'Petal_Length', 'Petal_Width'])\niris_data['target'] = iris.target\n\n# 3 数据集的划分\nX_train,X_test,y_train,y_test = train_test_split(iris.data,iris.target,test_size=0.2)\n\n# 4 归一化\ntransfer = StandardScaler()\nX_train = transfer.fit_transform(X_train)\nX_test = transfer.transform(X_test)\n\n# 5 机器学习 KNN\n# 5.1 实例化估计器\n#\n#不指定 n_neighbors ,使用网格搜索进行循环训练\nestimator = KNeighborsClassifier()\n# 5.2 模型调优 -- 交叉验证，网格搜素\nestimator = GridSearchCV(estimator,param_grid={\"n_neighbors\":[1,3,5,7]},cv=5) # 5 折\n# 5.2 进行训练\nestimator.fit(X_train,y_train)\n \n# 6 模型评估\ny_pred = estimator.predict(X_test)\nprint(\"预测值：\\n\",y_pred)\nprint(\"预测值与真实值是否相等：\\n\",y_pred==y_test)\naccuracy = accuracy_score(y_test, y_pred)\nprint(f\"\\nKNN 模型的准确率: {accuracy:.4f}\")\n\n# 交叉验证的相关参数\nprint(f\"最好结果：{estimator.best_score_}\")\nprint(f\"最好模型：{estimator.best_estimator_}\")\nprint(f\"最好模型结果：{estimator.cv_results_}\")\n```\n\n\n### 机器学习的基本步骤\n- 获取数据集\n- 数据集基本处理\n - 去重去空、填充等操作 \n - 确定特征值和目标值\n - 分割数据集\n- 特征工程（特征预处理 标准化等）\n- 机器学习\n- 模型评估\n\n### 数据分割的方法\n- 留出法\n 训练/测试集的划分要尽可能保持数据分布的一致性，避免因数据划分过程引入额外的偏差而对最终结果产生影响。\n 单次使用留出法得到的估计结果往往不够稳定可靠，在使用留出法时，一般要采用若干次随机划分、重复进行实验评估后取平均值作为留出法的评估结果。\n ``` python\n\tfrom sklearn.model_selection import KFold,StratifiedKFold\n\timport pandas as pd\n\tX = np.array([\n\t[1,2,3,4],\n\t[11,12,13,14],\n\t[21,22,23,24],\n\t[31,32,33,34],\n\t[41,42,43,44],\n\t[51,52,53,54],\n\t[61,62,63,64],\n\t[71,72,73,74]\n\t])\n\ty=np.array([1,1,0,0,1,1,0,0])\n\tfolder = KFold(n_splits=4)\n\tsfloder = StratifiedKFold(n_splits=4)\n\tprint(\"KFOLD:\")\n\tfor train,test in folder.split(X,y):\n\t print(f\"train:{train},test:{test}\")\n\tprint(\"SKFOLD:\")\n\tfor train,test in sfloder.split(X,y):\n print(f\"train:{train},test:{test}\")\n\t```\n\t\n- 自助法\n- 交叉验证法","slug":"machinelearning/knn","published":1,"updated":"2025-01-24T10:16:05.657Z","comments":1,"layout":"post","photos":[],"_id":"cmgsv0jnj000f3wah7exd6oi7","content":"k近邻算法（K-Nearest Neighbors）KNN 将当前样本的类别归类于距离最近的k 个样本的类别
\n距离公式(2维) \n欧式距离 \n曼哈顿距离 \n切比雪夫距离 \n \nk值选择问题 \n\n\nk值 \n影响 \n \n \n\n越大 \n模型过拟合，准确率波动较大 \n \n\n越小 \n模型欠拟合，准确率趋于稳定但可能较低 \n \n
\n特征预处理 \n通过一些转换函数将特征数据转换成更加适合算法模型的特征数据过程
\n \n\n归一化
\n1 from sklearn.preprocessing import MinMaxScaler
标准化
\n1 from sklearn.preprocessing import StandardScaler
\nKNN代码实现 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 import seaborn as snsimport matplotlib.pyplot as plt import pandas as pdfrom sklearn.model_selection import train_test_splitfrom sklearn.datasets import load_irisfrom sklearn.preprocessing import MinMaxScaler,StandardScalerfrom sklearn.neighbors import KNeighborsClassifierfrom sklearn.metrics import accuracy_scoreiris = load_iris() iris_data = pd.DataFrame(iris.data,columns=['Sepal_Length' , 'Sepal_Width' , 'Petal_Length' , 'Petal_Width' ]) iris_data['target' ] = iris.target def iris_plot (data,col1,col2 ): sns.lmplot(x=col1,y=col2,data=data,hue="target" ,fit_reg=False ) plt.show() X_train,X_test,y_train,y_test = train_test_split(iris.data,iris.target,test_size=0.2 ,random_state=44 ) transfer = StandardScaler() X_train = transfer.fit_transform(X_train) X_test = transfer.transform(X_test) estimator = KNeighborsClassifier(n_neighbors=9 ) estimator.fit(X_train,y_train) y_pred = estimator.predict(X_test) print ("预测值：\\n" ,y_pre)print ("预测值与真实值是否相等：\\n" ,y_pred==y_test)accuracy = accuracy_score(y_test, y_pred) print (f"\\nKNN 模型的准确率: {accuracy:.4 f} " )
\n交叉验证与网格搜索 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 import seaborn as snsimport matplotlib.pyplot as plt import pandas as pdfrom sklearn.model_selection import train_test_split,GridSearchCVfrom sklearn.datasets import load_irisfrom sklearn.preprocessing import MinMaxScaler,StandardScalerfrom sklearn.neighbors import KNeighborsClassifierfrom sklearn.metrics import accuracy_scoreiris = load_iris() iris_data = pd.DataFrame(iris.data,columns=['Sepal_Length' , 'Sepal_Width' , 'Petal_Length' , 'Petal_Width' ]) iris_data['target' ] = iris.target X_train,X_test,y_train,y_test = train_test_split(iris.data,iris.target,test_size=0.2 ) transfer = StandardScaler() X_train = transfer.fit_transform(X_train) X_test = transfer.transform(X_test) estimator = KNeighborsClassifier() estimator = GridSearchCV(estimator,param_grid={"n_neighbors" :[1 ,3 ,5 ,7 ]},cv=5 ) estimator.fit(X_train,y_train) y_pred = estimator.predict(X_test) print ("预测值：\\n" ,y_pred)print ("预测值与真实值是否相等：\\n" ,y_pred==y_test)accuracy = accuracy_score(y_test, y_pred) print (f"\\nKNN 模型的准确率: {accuracy:.4 f} " )print (f"最好结果：{estimator.best_score_} " )print (f"最好模型：{estimator.best_estimator_} " )print (f"最好模型结果：{estimator.cv_results_} " )
\n机器学习的基本步骤 \n获取数据集 \n数据集基本处理\n去重去空、填充等操作 \n确定特征值和目标值 \n分割数据集 \n \n \n特征工程（特征预处理 标准化等） \n机器学习 \n模型评估 \n \n数据分割的方法 \n留出法1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 from sklearn.model_selection import KFold,StratifiedKFoldimport pandas as pdX = np.array([ [1 ,2 ,3 ,4 ], [11 ,12 ,13 ,14 ], [21 ,22 ,23 ,24 ], [31 ,32 ,33 ,34 ], [41 ,42 ,43 ,44 ], [51 ,52 ,53 ,54 ], [61 ,62 ,63 ,64 ], [71 ,72 ,73 ,74 ] ]) y=np.array([1 ,1 ,0 ,0 ,1 ,1 ,0 ,0 ]) folder = KFold(n_splits=4 ) sfloder = StratifiedKFold(n_splits=4 ) print ("KFOLD:" )for train,test in folder.split(X,y): print (f"train:{train} ,test:{test} " ) print ("SKFOLD:" )for train,test in sfloder.split(X,y): print (f"train:{train} ,test:{test} " )
\n自助法 \n交叉验证法 \n \n","cover":false,"excerpt":"","more":"k近邻算法（K-Nearest Neighbors）KNN 将当前样本的类别归类于距离最近的k 个样本的类别
\n距离公式(2维) \n欧式距离 \n曼哈顿距离 \n切比雪夫距离 \n \nk值选择问题 \n\n\nk值 \n影响 \n \n \n\n越大 \n模型过拟合，准确率波动较大 \n \n\n越小 \n模型欠拟合，准确率趋于稳定但可能较低 \n \n
\n特征预处理 \n通过一些转换函数将特征数据转换成更加适合算法模型的特征数据过程
\n \n\n归一化
\n1 from sklearn.preprocessing import MinMaxScaler
标准化
\n1 from sklearn.preprocessing import StandardScaler
\nKNN代码实现 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 import seaborn as snsimport matplotlib.pyplot as plt import pandas as pdfrom sklearn.model_selection import train_test_splitfrom sklearn.datasets import load_irisfrom sklearn.preprocessing import MinMaxScaler,StandardScalerfrom sklearn.neighbors import KNeighborsClassifierfrom sklearn.metrics import accuracy_scoreiris = load_iris() iris_data = pd.DataFrame(iris.data,columns=['Sepal_Length' , 'Sepal_Width' , 'Petal_Length' , 'Petal_Width' ]) iris_data['target' ] = iris.target def iris_plot (data,col1,col2 ): sns.lmplot(x=col1,y=col2,data=data,hue="target" ,fit_reg=False ) plt.show() X_train,X_test,y_train,y_test = train_test_split(iris.data,iris.target,test_size=0.2 ,random_state=44 ) transfer = StandardScaler() X_train = transfer.fit_transform(X_train) X_test = transfer.transform(X_test) estimator = KNeighborsClassifier(n_neighbors=9 ) estimator.fit(X_train,y_train) y_pred = estimator.predict(X_test) print ("预测值：\\n" ,y_pre)print ("预测值与真实值是否相等：\\n" ,y_pred==y_test)accuracy = accuracy_score(y_test, y_pred) print (f"\\nKNN 模型的准确率: {accuracy:.4 f} " )
\n交叉验证与网格搜索 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 import seaborn as snsimport matplotlib.pyplot as plt import pandas as pdfrom sklearn.model_selection import train_test_split,GridSearchCVfrom sklearn.datasets import load_irisfrom sklearn.preprocessing import MinMaxScaler,StandardScalerfrom sklearn.neighbors import KNeighborsClassifierfrom sklearn.metrics import accuracy_scoreiris = load_iris() iris_data = pd.DataFrame(iris.data,columns=['Sepal_Length' , 'Sepal_Width' , 'Petal_Length' , 'Petal_Width' ]) iris_data['target' ] = iris.target X_train,X_test,y_train,y_test = train_test_split(iris.data,iris.target,test_size=0.2 ) transfer = StandardScaler() X_train = transfer.fit_transform(X_train) X_test = transfer.transform(X_test) estimator = KNeighborsClassifier() estimator = GridSearchCV(estimator,param_grid={"n_neighbors" :[1 ,3 ,5 ,7 ]},cv=5 ) estimator.fit(X_train,y_train) y_pred = estimator.predict(X_test) print ("预测值：\\n" ,y_pred)print ("预测值与真实值是否相等：\\n" ,y_pred==y_test)accuracy = accuracy_score(y_test, y_pred) print (f"\\nKNN 模型的准确率: {accuracy:.4 f} " )print (f"最好结果：{estimator.best_score_} " )print (f"最好模型：{estimator.best_estimator_} " )print (f"最好模型结果：{estimator.cv_results_} " )
\n机器学习的基本步骤 \n获取数据集 \n数据集基本处理\n去重去空、填充等操作 \n确定特征值和目标值 \n分割数据集 \n \n \n特征工程（特征预处理 标准化等） \n机器学习 \n模型评估 \n \n数据分割的方法 \n留出法1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 from sklearn.model_selection import KFold,StratifiedKFoldimport pandas as pdX = np.array([ [1 ,2 ,3 ,4 ], [11 ,12 ,13 ,14 ], [21 ,22 ,23 ,24 ], [31 ,32 ,33 ,34 ], [41 ,42 ,43 ,44 ], [51 ,52 ,53 ,54 ], [61 ,62 ,63 ,64 ], [71 ,72 ,73 ,74 ] ]) y=np.array([1 ,1 ,0 ,0 ,1 ,1 ,0 ,0 ]) folder = KFold(n_splits=4 ) sfloder = StratifiedKFold(n_splits=4 ) print ("KFOLD:" )for train,test in folder.split(X,y): print (f"train:{train} ,test:{test} " ) print ("SKFOLD:" )for train,test in sfloder.split(X,y): print (f"train:{train} ,test:{test} " )
\n自助法 \n交叉验证法 \n \n"},{"title":"线性回归","mathjax":true,"abbrlink":52662,"date":"2025-01-19T08:46:51.000Z","_content":"\n### 线性回归简介\n>用于预测一个连续的目标变量（因变量），与一个或多个特征（自变量）之间存在线性关系。\n\n假设函数： \n$$y = w_1x_1 + w_2x_2 + \\cdot\\cdot\\cdot+w_nx_n$$\n- $y$ 是目标变量（因变量），即我们希望预测的值。\n- $x1,x2,…,xn$ 是特征变量（自变量），即输入的值。\n### 损失函数\n\n为了找到最佳的线性模型，我们需要通过最小化损失函数来优化模型参数。在线性回归中，常用的损失函数是 **均方误差（MSE）**：\n$$J(\\theta) = \\frac{1}{2N} \\sum_{i=1}^{N} (y_i - f_\\theta(x_i))^2$$\n- N 是样本的数量。\n- $y_i$ 是第 i 个样本的真实值。\n- $f_\\theta(x_i)$ 是模型预测的第 i 个样本的值。\n\n### 线性回归优化\n\n- 梯度下降法\n ```python\n from sklearn.datasets import fetch_california_housing\n from sklearn.model_selection import train_test_split\n from sklearn.preprocessing import StandardScaler\n from sklearn.linear_model import SGDRegressor\n from sklearn.metrics import mean_squared_error\n\n # 1. 获取数据集\n housing = fetch_california_housing()\n\n # 2. 数据集处理\n # 2.1 分割数据集\n X_train, X_test, y_train, y_test = train_test_split(housing.data, housing.target, test_size=0.25)\n\n # 3. 特征工程\n # 3.1 标准化\n transfer = StandardScaler()\n X_train = transfer.fit_transform(X_train)\n X_test = transfer.transform(X_test) # 使用 transform() 而不是 fit_transform()\n\n # 4.机器学习- 梯度下降法\n estimater = SGDRegressor(max_iter=1000, eta0=0.01)\n estimater.fit(X_train, y_train)\n print(f\"SGD模型的偏置是：{estimater.intercept_}\")\n print(f\"SGD模型的系数是：{estimater.coef_}\")\n\n # 5. 模型评估\n y_pred = estimater.predict(X_test)\n print(f\"SGD模型预测值：{y_pred}\")\n mse = mean_squared_error(y_test, y_pred)\n print(f\"SGD模型均方误差:{mse}\")\n ```\n\n- 正规方程\n ```python\n from sklearn.datasets import fetch_california_housing\n from sklearn.model_selection import train_test_split\n from sklearn.preprocessing import StandardScaler\n from sklearn.linear_model import LinearRegression\n from sklearn.metrics import mean_squared_error\n\n # 1. 获取数据集\n housing = fetch_california_housing()\n\n # 2. 数据集处理\n # 2.1 分割数据集\n X_train, X_test, y_train, y_test = train_test_split(housing.data, housing.target, test_size=0.25)\n\n # 3. 特征工程\n # 3.1 标准化\n transfer = StandardScaler()\n X_train = transfer.fit_transform(X_train)\n X_test = transfer.fit_transform(X_test)\n\n # 4.机器学习- 线性回归\n estimater = LinearRegression()\n estimater.fit(X_train, y_train)\n print(f\"模型的偏置是：{estimater.intercept_}\")\n print(f\"模型的系数是：{estimater.coef_}\")\n\n # 5. 模型评估\n y_pred = estimater.predict(X_test)\n print(f\"模型预测值：{y_pred}\")\n mse = mean_squared_error(y_test, y_pred)\n print(f\"模型均方误差:{mse}\")\n ```\n\n- 岭回归\n ```python\n from sklearn.datasets import fetch_california_housing\n from sklearn.model_selection import train_test_split\n from sklearn.preprocessing import StandardScaler\n from sklearn.linear_model import Ridge, RidgeCV\n from sklearn.metrics import mean_squared_error\n\n # 1. 获取数据集\n housing = fetch_california_housing()\n\n # 2. 数据集处理\n # 2.1 分割数据集\n X_train, X_test, y_train, y_test = train_test_split(housing.data, housing.target, test_size=0.25)\n\n # 3. 特征工程\n # 3.1 标准化\n transfer = StandardScaler()\n X_train = transfer.fit_transform(X_train)\n X_test = transfer.transform(X_test) # 使用 transform() 而不是 fit_transform()\n\n # 4.机器学习- 岭回归 使用了Ridge的alpha的搜索\n # estimater = Ridge(alpha=1.0)\n estimater = RidgeCV(alphas=[0.001, 0.01, 0.1, 1, 10, 100])\n estimater.fit(X_train, y_train)\n print(f\"Ridge模型的偏置是：{estimater.intercept_}\")\n print(f\"Ridge模型的系数是：{estimater.coef_}\")\n\n # 查看最佳 alpha\n print(f\"最佳 alpha 值是：{estimater.alpha_}\")\n\n # 5. 模型评估\n y_pred = estimater.predict(X_test)\n print(f\"Ridge模型预测值：{y_pred}\")\n mse = mean_squared_error(y_test, y_pred)\n print(f\"Ridge模型均方误差:{mse}\")\n ```\n\n这样每个代码块的缩进保持一致，便于阅读和理解。如果有其他优化需求，随时告诉我！\n\n\n\n\n\n### 模型保存和加载\n```python\nfrom sklearn.datasets import fetch_california_housing\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import Ridge, RidgeCV\nfrom sklearn.metrics import mean_squared_error\nimport joblib\n\ndef save_model():\n # 1. 获取数据集\n housing = fetch_california_housing()\n # 2. 数据集处理\n # 2.1 分割数据集\n X_train, X_test, y_train, y_test = train_test_split(housing.data, housing.target, test_size=0.25)\n # 3. 特征工程\n # 3.1 标准化\n transfer = StandardScaler()\n X_train = transfer.fit_transform(X_train)\n X_test = transfer.transform(X_test) # 使用 transform() 而不是 fit_transform()\n # 4. 机器学习 - 岭回归 使用了Ridge的alpha的搜索\n estimater = RidgeCV(alphas=[0.001, 0.01, 0.1, 1, 10, 100])\n estimater.fit(X_train, y_train)\n print(f\"Ridge模型的偏置是：{estimater.intercept_}\")\n print(f\"Ridge模型的系数是：{estimater.coef_}\")\n # 保存模型\n joblib.dump(estimater, 'ridge_model.pkl')\n # 查看最佳 alpha\n print(f\"最佳 alpha 值是：{estimater.alpha_}\")\n # 5. 模型评估\n y_pred = estimater.predict(X_test)\n mse = mean_squared_error(y_test, y_pred)\n print(f\"Ridge模型均方误差:{mse}\")\n\ndef load_model():\n # 1. 获取数据集\n housing = fetch_california_housing()\n # 2. 数据集处理\n # 2.1 分割数据集\n X_train, X_test, y_train, y_test = train_test_split(housing.data, housing.target, test_size=0.25)\n # 3. 特征工程\n # 3.1 标准化\n transfer = StandardScaler()\n X_train = transfer.fit_transform(X_train)\n X_test = transfer.transform(X_test) # 使用 transform() 而不是 fit_transform()\n # 加载模型\n estimater = joblib.load('ridge_model.pkl')\n print(f\"Ridge模型的偏置是：{estimater.intercept_}\")\n print(f\"Ridge模型的系数是：{estimater.coef_}\")\n # 查看最佳 alpha\n print(f\"最佳 alpha 值是：{estimater.alpha_}\")\n # 5. 模型评估\n y_pred = estimater.predict(X_test)\n mse = mean_squared_error(y_test, y_pred)\n print(f\"Ridge模型预测值：{y_pred}\")\n print(f\"Ridge模型均方误差:{mse}\")\n\nprint(\"训练并保存模型：\")\nsave_model()\nprint(\"加载模型\")\nload_model()\n```","source":"_posts/machinelearning/linearreression.md","raw":"---\ntitle: 线性回归\ntags: linear-regression\ncategories: machinelearning\nmathjax: true\nabbrlink: 52662\ndate: 2025-01-19 16:46:51\n---\n\n### 线性回归简介\n>用于预测一个连续的目标变量（因变量），与一个或多个特征（自变量）之间存在线性关系。\n\n假设函数： \n$$y = w_1x_1 + w_2x_2 + \\cdot\\cdot\\cdot+w_nx_n$$\n- $y$ 是目标变量（因变量），即我们希望预测的值。\n- $x1,x2,…,xn$ 是特征变量（自变量），即输入的值。\n### 损失函数\n\n为了找到最佳的线性模型，我们需要通过最小化损失函数来优化模型参数。在线性回归中，常用的损失函数是 **均方误差（MSE）**：\n$$J(\\theta) = \\frac{1}{2N} \\sum_{i=1}^{N} (y_i - f_\\theta(x_i))^2$$\n- N 是样本的数量。\n- $y_i$ 是第 i 个样本的真实值。\n- $f_\\theta(x_i)$ 是模型预测的第 i 个样本的值。\n\n### 线性回归优化\n\n- 梯度下降法\n ```python\n from sklearn.datasets import fetch_california_housing\n from sklearn.model_selection import train_test_split\n from sklearn.preprocessing import StandardScaler\n from sklearn.linear_model import SGDRegressor\n from sklearn.metrics import mean_squared_error\n\n # 1. 获取数据集\n housing = fetch_california_housing()\n\n # 2. 数据集处理\n # 2.1 分割数据集\n X_train, X_test, y_train, y_test = train_test_split(housing.data, housing.target, test_size=0.25)\n\n # 3. 特征工程\n # 3.1 标准化\n transfer = StandardScaler()\n X_train = transfer.fit_transform(X_train)\n X_test = transfer.transform(X_test) # 使用 transform() 而不是 fit_transform()\n\n # 4.机器学习- 梯度下降法\n estimater = SGDRegressor(max_iter=1000, eta0=0.01)\n estimater.fit(X_train, y_train)\n print(f\"SGD模型的偏置是：{estimater.intercept_}\")\n print(f\"SGD模型的系数是：{estimater.coef_}\")\n\n # 5. 模型评估\n y_pred = estimater.predict(X_test)\n print(f\"SGD模型预测值：{y_pred}\")\n mse = mean_squared_error(y_test, y_pred)\n print(f\"SGD模型均方误差:{mse}\")\n ```\n\n- 正规方程\n ```python\n from sklearn.datasets import fetch_california_housing\n from sklearn.model_selection import train_test_split\n from sklearn.preprocessing import StandardScaler\n from sklearn.linear_model import LinearRegression\n from sklearn.metrics import mean_squared_error\n\n # 1. 获取数据集\n housing = fetch_california_housing()\n\n # 2. 数据集处理\n # 2.1 分割数据集\n X_train, X_test, y_train, y_test = train_test_split(housing.data, housing.target, test_size=0.25)\n\n # 3. 特征工程\n # 3.1 标准化\n transfer = StandardScaler()\n X_train = transfer.fit_transform(X_train)\n X_test = transfer.fit_transform(X_test)\n\n # 4.机器学习- 线性回归\n estimater = LinearRegression()\n estimater.fit(X_train, y_train)\n print(f\"模型的偏置是：{estimater.intercept_}\")\n print(f\"模型的系数是：{estimater.coef_}\")\n\n # 5. 模型评估\n y_pred = estimater.predict(X_test)\n print(f\"模型预测值：{y_pred}\")\n mse = mean_squared_error(y_test, y_pred)\n print(f\"模型均方误差:{mse}\")\n ```\n\n- 岭回归\n ```python\n from sklearn.datasets import fetch_california_housing\n from sklearn.model_selection import train_test_split\n from sklearn.preprocessing import StandardScaler\n from sklearn.linear_model import Ridge, RidgeCV\n from sklearn.metrics import mean_squared_error\n\n # 1. 获取数据集\n housing = fetch_california_housing()\n\n # 2. 数据集处理\n # 2.1 分割数据集\n X_train, X_test, y_train, y_test = train_test_split(housing.data, housing.target, test_size=0.25)\n\n # 3. 特征工程\n # 3.1 标准化\n transfer = StandardScaler()\n X_train = transfer.fit_transform(X_train)\n X_test = transfer.transform(X_test) # 使用 transform() 而不是 fit_transform()\n\n # 4.机器学习- 岭回归 使用了Ridge的alpha的搜索\n # estimater = Ridge(alpha=1.0)\n estimater = RidgeCV(alphas=[0.001, 0.01, 0.1, 1, 10, 100])\n estimater.fit(X_train, y_train)\n print(f\"Ridge模型的偏置是：{estimater.intercept_}\")\n print(f\"Ridge模型的系数是：{estimater.coef_}\")\n\n # 查看最佳 alpha\n print(f\"最佳 alpha 值是：{estimater.alpha_}\")\n\n # 5. 模型评估\n y_pred = estimater.predict(X_test)\n print(f\"Ridge模型预测值：{y_pred}\")\n mse = mean_squared_error(y_test, y_pred)\n print(f\"Ridge模型均方误差:{mse}\")\n ```\n\n这样每个代码块的缩进保持一致，便于阅读和理解。如果有其他优化需求，随时告诉我！\n\n\n\n\n\n### 模型保存和加载\n```python\nfrom sklearn.datasets import fetch_california_housing\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import Ridge, RidgeCV\nfrom sklearn.metrics import mean_squared_error\nimport joblib\n\ndef save_model():\n # 1. 获取数据集\n housing = fetch_california_housing()\n # 2. 数据集处理\n # 2.1 分割数据集\n X_train, X_test, y_train, y_test = train_test_split(housing.data, housing.target, test_size=0.25)\n # 3. 特征工程\n # 3.1 标准化\n transfer = StandardScaler()\n X_train = transfer.fit_transform(X_train)\n X_test = transfer.transform(X_test) # 使用 transform() 而不是 fit_transform()\n # 4. 机器学习 - 岭回归 使用了Ridge的alpha的搜索\n estimater = RidgeCV(alphas=[0.001, 0.01, 0.1, 1, 10, 100])\n estimater.fit(X_train, y_train)\n print(f\"Ridge模型的偏置是：{estimater.intercept_}\")\n print(f\"Ridge模型的系数是：{estimater.coef_}\")\n # 保存模型\n joblib.dump(estimater, 'ridge_model.pkl')\n # 查看最佳 alpha\n print(f\"最佳 alpha 值是：{estimater.alpha_}\")\n # 5. 模型评估\n y_pred = estimater.predict(X_test)\n mse = mean_squared_error(y_test, y_pred)\n print(f\"Ridge模型均方误差:{mse}\")\n\ndef load_model():\n # 1. 获取数据集\n housing = fetch_california_housing()\n # 2. 数据集处理\n # 2.1 分割数据集\n X_train, X_test, y_train, y_test = train_test_split(housing.data, housing.target, test_size=0.25)\n # 3. 特征工程\n # 3.1 标准化\n transfer = StandardScaler()\n X_train = transfer.fit_transform(X_train)\n X_test = transfer.transform(X_test) # 使用 transform() 而不是 fit_transform()\n # 加载模型\n estimater = joblib.load('ridge_model.pkl')\n print(f\"Ridge模型的偏置是：{estimater.intercept_}\")\n print(f\"Ridge模型的系数是：{estimater.coef_}\")\n # 查看最佳 alpha\n print(f\"最佳 alpha 值是：{estimater.alpha_}\")\n # 5. 模型评估\n y_pred = estimater.predict(X_test)\n mse = mean_squared_error(y_test, y_pred)\n print(f\"Ridge模型预测值：{y_pred}\")\n print(f\"Ridge模型均方误差:{mse}\")\n\nprint(\"训练并保存模型：\")\nsave_model()\nprint(\"加载模型\")\nload_model()\n```","slug":"machinelearning/linearreression","published":1,"updated":"2025-02-17T08:31:26.908Z","comments":1,"layout":"post","photos":[],"_id":"cmgsv0jnk000g3wah2r7mcppb","content":"线性回归简介 \n用于预测一个连续的目标变量（因变量），与一个或多个特征（自变量）之间存在线性关系。
\n \n假设函数：
\n\n$y$ 是目标变量（因变量），即我们希望预测的值。 \n$x1,x2,…,xn$ 是特征变量（自变量），即输入的值。 \n \n损失函数 为了找到最佳的线性模型，我们需要通过最小化损失函数来优化模型参数。在线性回归中，常用的损失函数是 均方误差（MSE） ：
\n\nN 是样本的数量。 \n$y_i$ 是第 i 个样本的真实值。 \n$f_\\theta(x_i)$ 是模型预测的第 i 个样本的值。 \n \n线性回归优化 \n梯度下降法
\n 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 from sklearn.datasets import fetch_california_housingfrom sklearn.model_selection import train_test_splitfrom sklearn.preprocessing import StandardScalerfrom sklearn.linear_model import SGDRegressorfrom sklearn.metrics import mean_squared_errorhousing = fetch_california_housing() X_train, X_test, y_train, y_test = train_test_split(housing.data, housing.target, test_size=0.25 ) transfer = StandardScaler() X_train = transfer.fit_transform(X_train) X_test = transfer.transform(X_test) estimater = SGDRegressor(max_iter=1000 , eta0=0.01 ) estimater.fit(X_train, y_train) print (f"SGD模型的偏置是：{estimater.intercept_} " )print (f"SGD模型的系数是：{estimater.coef_} " )y_pred = estimater.predict(X_test) print (f"SGD模型预测值：{y_pred} " )mse = mean_squared_error(y_test, y_pred) print (f"SGD模型均方误差:{mse} " )
正规方程
\n 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 from sklearn.datasets import fetch_california_housingfrom sklearn.model_selection import train_test_splitfrom sklearn.preprocessing import StandardScalerfrom sklearn.linear_model import LinearRegressionfrom sklearn.metrics import mean_squared_errorhousing = fetch_california_housing() X_train, X_test, y_train, y_test = train_test_split(housing.data, housing.target, test_size=0.25 ) transfer = StandardScaler() X_train = transfer.fit_transform(X_train) X_test = transfer.fit_transform(X_test) estimater = LinearRegression() estimater.fit(X_train, y_train) print (f"模型的偏置是：{estimater.intercept_} " )print (f"模型的系数是：{estimater.coef_} " )y_pred = estimater.predict(X_test) print (f"模型预测值：{y_pred} " )mse = mean_squared_error(y_test, y_pred) print (f"模型均方误差:{mse} " )
岭回归
\n 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 from sklearn.datasets import fetch_california_housingfrom sklearn.model_selection import train_test_splitfrom sklearn.preprocessing import StandardScalerfrom sklearn.linear_model import Ridge, RidgeCVfrom sklearn.metrics import mean_squared_errorhousing = fetch_california_housing() X_train, X_test, y_train, y_test = train_test_split(housing.data, housing.target, test_size=0.25 ) transfer = StandardScaler() X_train = transfer.fit_transform(X_train) X_test = transfer.transform(X_test) estimater = RidgeCV(alphas=[0.001 , 0.01 , 0.1 , 1 , 10 , 100 ]) estimater.fit(X_train, y_train) print (f"Ridge模型的偏置是：{estimater.intercept_} " )print (f"Ridge模型的系数是：{estimater.coef_} " )print (f"最佳 alpha 值是：{estimater.alpha_} " )y_pred = estimater.predict(X_test) print (f"Ridge模型预测值：{y_pred} " )mse = mean_squared_error(y_test, y_pred) print (f"Ridge模型均方误差:{mse} " )
\n这样每个代码块的缩进保持一致，便于阅读和理解。如果有其他优化需求，随时告诉我！
\n
\n
\n模型保存和加载 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 from sklearn.datasets import fetch_california_housingfrom sklearn.model_selection import train_test_splitfrom sklearn.preprocessing import StandardScalerfrom sklearn.linear_model import Ridge, RidgeCVfrom sklearn.metrics import mean_squared_errorimport joblibdef save_model (): housing = fetch_california_housing() X_train, X_test, y_train, y_test = train_test_split(housing.data, housing.target, test_size=0.25 ) transfer = StandardScaler() X_train = transfer.fit_transform(X_train) X_test = transfer.transform(X_test) estimater = RidgeCV(alphas=[0.001 , 0.01 , 0.1 , 1 , 10 , 100 ]) estimater.fit(X_train, y_train) print (f"Ridge模型的偏置是：{estimater.intercept_} " ) print (f"Ridge模型的系数是：{estimater.coef_} " ) joblib.dump(estimater, 'ridge_model.pkl' ) print (f"最佳 alpha 值是：{estimater.alpha_} " ) y_pred = estimater.predict(X_test) mse = mean_squared_error(y_test, y_pred) print (f"Ridge模型均方误差:{mse} " ) def load_model (): housing = fetch_california_housing() X_train, X_test, y_train, y_test = train_test_split(housing.data, housing.target, test_size=0.25 ) transfer = StandardScaler() X_train = transfer.fit_transform(X_train) X_test = transfer.transform(X_test) estimater = joblib.load('ridge_model.pkl' ) print (f"Ridge模型的偏置是：{estimater.intercept_} " ) print (f"Ridge模型的系数是：{estimater.coef_} " ) print (f"最佳 alpha 值是：{estimater.alpha_} " ) y_pred = estimater.predict(X_test) mse = mean_squared_error(y_test, y_pred) print (f"Ridge模型预测值：{y_pred} " ) print (f"Ridge模型均方误差:{mse} " ) print ("训练并保存模型：" )save_model() print ("加载模型" )load_model()
线性回归简介 \n用于预测一个连续的目标变量（因变量），与一个或多个特征（自变量）之间存在线性关系。
\n \n假设函数：
\n\n$y$ 是目标变量（因变量），即我们希望预测的值。 \n$x1,x2,…,xn$ 是特征变量（自变量），即输入的值。 \n \n损失函数 为了找到最佳的线性模型，我们需要通过最小化损失函数来优化模型参数。在线性回归中，常用的损失函数是 均方误差（MSE） ：
\n\nN 是样本的数量。 \n$y_i$ 是第 i 个样本的真实值。 \n$f_\\theta(x_i)$ 是模型预测的第 i 个样本的值。 \n \n线性回归优化 \n梯度下降法
\n 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 from sklearn.datasets import fetch_california_housingfrom sklearn.model_selection import train_test_splitfrom sklearn.preprocessing import StandardScalerfrom sklearn.linear_model import SGDRegressorfrom sklearn.metrics import mean_squared_errorhousing = fetch_california_housing() X_train, X_test, y_train, y_test = train_test_split(housing.data, housing.target, test_size=0.25 ) transfer = StandardScaler() X_train = transfer.fit_transform(X_train) X_test = transfer.transform(X_test) estimater = SGDRegressor(max_iter=1000 , eta0=0.01 ) estimater.fit(X_train, y_train) print (f"SGD模型的偏置是：{estimater.intercept_} " )print (f"SGD模型的系数是：{estimater.coef_} " )y_pred = estimater.predict(X_test) print (f"SGD模型预测值：{y_pred} " )mse = mean_squared_error(y_test, y_pred) print (f"SGD模型均方误差:{mse} " )
正规方程
\n 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 from sklearn.datasets import fetch_california_housingfrom sklearn.model_selection import train_test_splitfrom sklearn.preprocessing import StandardScalerfrom sklearn.linear_model import LinearRegressionfrom sklearn.metrics import mean_squared_errorhousing = fetch_california_housing() X_train, X_test, y_train, y_test = train_test_split(housing.data, housing.target, test_size=0.25 ) transfer = StandardScaler() X_train = transfer.fit_transform(X_train) X_test = transfer.fit_transform(X_test) estimater = LinearRegression() estimater.fit(X_train, y_train) print (f"模型的偏置是：{estimater.intercept_} " )print (f"模型的系数是：{estimater.coef_} " )y_pred = estimater.predict(X_test) print (f"模型预测值：{y_pred} " )mse = mean_squared_error(y_test, y_pred) print (f"模型均方误差:{mse} " )
岭回归
\n 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 from sklearn.datasets import fetch_california_housingfrom sklearn.model_selection import train_test_splitfrom sklearn.preprocessing import StandardScalerfrom sklearn.linear_model import Ridge, RidgeCVfrom sklearn.metrics import mean_squared_errorhousing = fetch_california_housing() X_train, X_test, y_train, y_test = train_test_split(housing.data, housing.target, test_size=0.25 ) transfer = StandardScaler() X_train = transfer.fit_transform(X_train) X_test = transfer.transform(X_test) estimater = RidgeCV(alphas=[0.001 , 0.01 , 0.1 , 1 , 10 , 100 ]) estimater.fit(X_train, y_train) print (f"Ridge模型的偏置是：{estimater.intercept_} " )print (f"Ridge模型的系数是：{estimater.coef_} " )print (f"最佳 alpha 值是：{estimater.alpha_} " )y_pred = estimater.predict(X_test) print (f"Ridge模型预测值：{y_pred} " )mse = mean_squared_error(y_test, y_pred) print (f"Ridge模型均方误差:{mse} " )
\n这样每个代码块的缩进保持一致，便于阅读和理解。如果有其他优化需求，随时告诉我！
\n
\n
\n模型保存和加载 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 from sklearn.datasets import fetch_california_housingfrom sklearn.model_selection import train_test_splitfrom sklearn.preprocessing import StandardScalerfrom sklearn.linear_model import Ridge, RidgeCVfrom sklearn.metrics import mean_squared_errorimport joblibdef save_model (): housing = fetch_california_housing() X_train, X_test, y_train, y_test = train_test_split(housing.data, housing.target, test_size=0.25 ) transfer = StandardScaler() X_train = transfer.fit_transform(X_train) X_test = transfer.transform(X_test) estimater = RidgeCV(alphas=[0.001 , 0.01 , 0.1 , 1 , 10 , 100 ]) estimater.fit(X_train, y_train) print (f"Ridge模型的偏置是：{estimater.intercept_} " ) print (f"Ridge模型的系数是：{estimater.coef_} " ) joblib.dump(estimater, 'ridge_model.pkl' ) print (f"最佳 alpha 值是：{estimater.alpha_} " ) y_pred = estimater.predict(X_test) mse = mean_squared_error(y_test, y_pred) print (f"Ridge模型均方误差:{mse} " ) def load_model (): housing = fetch_california_housing() X_train, X_test, y_train, y_test = train_test_split(housing.data, housing.target, test_size=0.25 ) transfer = StandardScaler() X_train = transfer.fit_transform(X_train) X_test = transfer.transform(X_test) estimater = joblib.load('ridge_model.pkl' ) print (f"Ridge模型的偏置是：{estimater.intercept_} " ) print (f"Ridge模型的系数是：{estimater.coef_} " ) print (f"最佳 alpha 值是：{estimater.alpha_} " ) y_pred = estimater.predict(X_test) mse = mean_squared_error(y_test, y_pred) print (f"Ridge模型预测值：{y_pred} " ) print (f"Ridge模型均方误差:{mse} " ) print ("训练并保存模型：" )save_model() print ("加载模型" )load_model()
logistic regression code 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 import pandas as pdimport numpy as npfrom sklearn.datasets import load_breast_cancerfrom sklearn.model_selection import train_test_splitfrom sklearn.preprocessing import StandardScalerfrom sklearn.linear_model import LogisticRegressiondata = load_breast_cancer() df = pd.DataFrame(data.data, columns=data.feature_names) df['target' ] = data.target for i in df.columns: if np.any (pd.isnull(df[i])): print (f"Filling missing values in column: {i} " ) X = df.iloc[:,0 :df.shape[1 ] - 1 ] y = df.loc[:,"target" ] X_train,X_test,y_train,y_test = train_test_split(X,y,test_size=0.3 ) df.head(1 ) transfer = StandardScaler() X_train = transfer.fit_transform(X_train) X_test = transfer.transform(X_test) estimator = LogisticRegression() estimator.fit(X_train,y_train) print (f"模型准确率：{estimator.score(X_test,y_test)} " )print (f"模型预测值为:\\n{estimator.predict(X_test)} " )
分类评估的参数 \n准确率
\n精准率
\n召回率
\nF1-score
\nroc曲线
\n \n1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 import pandas as pdimport numpy as npfrom sklearn.datasets import load_breast_cancerfrom sklearn.model_selection import train_test_splitfrom sklearn.preprocessing import StandardScalerfrom sklearn.linear_model import LogisticRegressionfrom sklearn.metrics import classification_report, roc_auc_scoredata = load_breast_cancer() df = pd.DataFrame(data.data, columns=data.feature_names) df['target' ] = data.target for i in df.columns: if np.any (pd.isnull(df[i])): print (f"Filling missing values in column: {i} " ) X = df.iloc[:, 0 :df.shape[1 ] - 1 ] y = df.loc[:, "target" ] X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3 ) df.head(1 ) transfer = StandardScaler() X_train = transfer.fit_transform(X_train) X_test = transfer.transform(X_test) estimator = LogisticRegression() estimator.fit(X_train, y_train) print (f"模型准确率：{estimator.score(X_test, y_test)} " )y_pred = estimator.predict(X_test) print (f"模型预测值为:\\n{y_pred} " )ret = classification_report(y_test, y_pred, labels=[1 , 0 ], target_names=["良性" , "恶性" ]) roc_score = roc_auc_score(y_test, y_pred) print (f"准确率、召回率：{ret} " )print (f"roc_score:{roc_score} " )
类别不平衡的处理 先准备类别不平衡的数据
\n1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 from imblearn.over_sampling import RandomOverSampler,SMOTEfrom imblearn.under_sampling import RandomUnderSamplerfrom sklearn.datasets import make_classificationimport matplotlib.pyplot as pltfrom collections import CounterX, y = make_classification( n_samples=5000 , n_features=2 , n_informative=2 , n_redundant=0 , n_repeated=0 , n_classes=3 , n_clusters_per_class=1 , weights=[0.01 , 0.05 , 0.94 ], random_state=0 , ) counter = Counter(y) plt.scatter(X[:,0 ],X[:,1 ],c=y) plt.show()
\n过采样\n随机过采样（RandomOverSampler)1 2 3 4 5 ros = RandomOverSampler() X_resampled,y_resampled = ros.fit_resample(X,y) print (Counter(y_resampled))plt.scatter(X_resampled[:,0 ],X_resampled[:,1 ],c=y_resampled) plt.show()
\nSMOTE过采样（SMOTE）1 2 3 4 5 smote = SMOTE() X_resampled,y_resampled = smote.fit_resample(X,y) print (Counter(y_resampled))plt.scatter(X_resampled[:,0 ],X_resampled[:,1 ],c=y_resampled) plt.show()
\n \n欠采样\n随机欠采样（RandomUnderSampler）1 2 3 4 5 rus = RandomUnderSampler(random_state=0 ) X_resampled,y_resampled = rus.fit_resample(X,y) print (Counter(y_resampled))plt.scatter(X_resampled[:,0 ],X_resampled[:,1 ],c=y_resampled) plt.show()
\n \n \n \n","cover":false,"excerpt":"","more":"logistic regression code 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 import pandas as pdimport numpy as npfrom sklearn.datasets import load_breast_cancerfrom sklearn.model_selection import train_test_splitfrom sklearn.preprocessing import StandardScalerfrom sklearn.linear_model import LogisticRegressiondata = load_breast_cancer() df = pd.DataFrame(data.data, columns=data.feature_names) df['target' ] = data.target for i in df.columns: if np.any (pd.isnull(df[i])): print (f"Filling missing values in column: {i} " ) X = df.iloc[:,0 :df.shape[1 ] - 1 ] y = df.loc[:,"target" ] X_train,X_test,y_train,y_test = train_test_split(X,y,test_size=0.3 ) df.head(1 ) transfer = StandardScaler() X_train = transfer.fit_transform(X_train) X_test = transfer.transform(X_test) estimator = LogisticRegression() estimator.fit(X_train,y_train) print (f"模型准确率：{estimator.score(X_test,y_test)} " )print (f"模型预测值为:\\n{estimator.predict(X_test)} " )
分类评估的参数 \n准确率
\n精准率
\n召回率
\nF1-score
\nroc曲线
\n \n1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 import pandas as pdimport numpy as npfrom sklearn.datasets import load_breast_cancerfrom sklearn.model_selection import train_test_splitfrom sklearn.preprocessing import StandardScalerfrom sklearn.linear_model import LogisticRegressionfrom sklearn.metrics import classification_report, roc_auc_scoredata = load_breast_cancer() df = pd.DataFrame(data.data, columns=data.feature_names) df['target' ] = data.target for i in df.columns: if np.any (pd.isnull(df[i])): print (f"Filling missing values in column: {i} " ) X = df.iloc[:, 0 :df.shape[1 ] - 1 ] y = df.loc[:, "target" ] X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3 ) df.head(1 ) transfer = StandardScaler() X_train = transfer.fit_transform(X_train) X_test = transfer.transform(X_test) estimator = LogisticRegression() estimator.fit(X_train, y_train) print (f"模型准确率：{estimator.score(X_test, y_test)} " )y_pred = estimator.predict(X_test) print (f"模型预测值为:\\n{y_pred} " )ret = classification_report(y_test, y_pred, labels=[1 , 0 ], target_names=["良性" , "恶性" ]) roc_score = roc_auc_score(y_test, y_pred) print (f"准确率、召回率：{ret} " )print (f"roc_score:{roc_score} " )
类别不平衡的处理 先准备类别不平衡的数据
\n1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 from imblearn.over_sampling import RandomOverSampler,SMOTEfrom imblearn.under_sampling import RandomUnderSamplerfrom sklearn.datasets import make_classificationimport matplotlib.pyplot as pltfrom collections import CounterX, y = make_classification( n_samples=5000 , n_features=2 , n_informative=2 , n_redundant=0 , n_repeated=0 , n_classes=3 , n_clusters_per_class=1 , weights=[0.01 , 0.05 , 0.94 ], random_state=0 , ) counter = Counter(y) plt.scatter(X[:,0 ],X[:,1 ],c=y) plt.show()
\n过采样\n随机过采样（RandomOverSampler)1 2 3 4 5 ros = RandomOverSampler() X_resampled,y_resampled = ros.fit_resample(X,y) print (Counter(y_resampled))plt.scatter(X_resampled[:,0 ],X_resampled[:,1 ],c=y_resampled) plt.show()
\nSMOTE过采样（SMOTE）1 2 3 4 5 smote = SMOTE() X_resampled,y_resampled = smote.fit_resample(X,y) print (Counter(y_resampled))plt.scatter(X_resampled[:,0 ],X_resampled[:,1 ],c=y_resampled) plt.show()
\n \n欠采样\n随机欠采样（RandomUnderSampler）1 2 3 4 5 rus = RandomUnderSampler(random_state=0 ) X_resampled,y_resampled = rus.fit_resample(X,y) print (Counter(y_resampled))plt.scatter(X_resampled[:,0 ],X_resampled[:,1 ],c=y_resampled) plt.show()
\n \n \n \n"},{"title":"组件使用","abbrlink":33957,"date":"2024-08-05T06:07:01.000Z","_content":"\n### 组件自动导入\n```json\n\t\"easycom\":{\n\t \"autoscan\": true,\n\t \"custom\": {\n\t \"^tui-(.*)\": \"@/components/thorui/tui-$1/tui-$1.vue\" // 匹配components目录内的vue文件\n\t }\n\t}\n```\n\n### `tui-sticky 吸顶容器` \n\n> 包含 以下 `tui` 组件 :\n> - tui-sticky\n> - tui-list-view\n> - tui-list-cell\n> \n\n```html\n\n \n \n \n 设置 \n \n \n \n \n \n \n \n \n \n \n \n \n\n\n```\n","source":"_posts/frontend/uniapp/component1.md","raw":"---\ntitle: 组件使用\ntags: uniapp\nabbrlink: 33957\ndate: 2024-08-05 14:07:01\n---\n\n### 组件自动导入\n```json\n\t\"easycom\":{\n\t \"autoscan\": true,\n\t \"custom\": {\n\t \"^tui-(.*)\": \"@/components/thorui/tui-$1/tui-$1.vue\" // 匹配components目录内的vue文件\n\t }\n\t}\n```\n\n### `tui-sticky 吸顶容器` \n\n> 包含 以下 `tui` 组件 :\n> - tui-sticky\n> - tui-list-view\n> - tui-list-cell\n> \n\n```html\n\n \n \n \n 设置 \n \n \n \n \n \n \n \n \n \n \n \n \n\n\n```\n","slug":"frontend/uniapp/component1","published":1,"updated":"2024-08-09T12:21:50.042Z","comments":1,"layout":"post","photos":[],"_id":"cmgsv0jno00103wahaue5bi4q","content":"组件自动导入 1 2 3 4 5 6 "easycom" : { "autoscan" : true , "custom" : { "^tui-(.*)" : "@/components/thorui/tui-$1/tui-$1.vue" } }
tui-sticky 吸顶容器\n包含 以下 tui 组件 :
\n\ntui-sticky \ntui-list-view \ntui-list-cell \n \n \n1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 <tui-sticky :scrollTop ="scrollTop" stickyHeight ="104rpx" container > <template v-slot:header > <view class ="sticky-item" > <view class ="setting" > 设置</view > </view > </template > <template v-slot:content > <tui-list-view class ="content" > <tui-list-cell :arrow ="false" > <switch class ='switch' checked color ="#FFCC33" /> </tui-list-cell > </tui-list-view > </template > </tui-sticky > <script setup > import { ref } from 'vue' import { onPageScroll } from '@dcloudio/uni-app' const scrollTop = ref (0 ) onPageScroll ((e ) => { scrollTop.value = e.scrollTop }) </script >
组件自动导入 1 2 3 4 5 6 "easycom" : { "autoscan" : true , "custom" : { "^tui-(.*)" : "@/components/thorui/tui-$1/tui-$1.vue" } }
tui-sticky 吸顶容器\n包含 以下 tui 组件 :
\n\ntui-sticky \ntui-list-view \ntui-list-cell \n \n \n1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 <tui-sticky :scrollTop ="scrollTop" stickyHeight ="104rpx" container > <template v-slot:header > <view class ="sticky-item" > <view class ="setting" > 设置</view > </view > </template > <template v-slot:content > <tui-list-view class ="content" > <tui-list-cell :arrow ="false" > <switch class ='switch' checked color ="#FFCC33" /> </tui-list-cell > </tui-list-view > </template > </tui-sticky > <script setup > import { ref } from 'vue' import { onPageScroll } from '@dcloudio/uni-app' const scrollTop = ref (0 ) onPageScroll ((e ) => { scrollTop.value = e.scrollTop }) </script >
防火墙关闭 1 2 3 systemctl stop firewalld systemctl disable firewalld
配置yum源 \n下载 repo 文件：Centos-7.repo /tmp，进入到/tmp \n备份并且替换系统的repo文件1 2 3 4 \tcp Centos-7.repo /etc/yum.repos.d/ cd /etc/yum.repos.d/ mv CentOS-Base.repo CentOS-Base.repo.bak mv Centos-7.repo CentOS-Base.repo
\n将nn1上的CentOS-Base.repo拷贝到其他主机1 2 3 4 5 scp /etc/yum.repos.d/CentOS-Base.repo root@nn2:/etc/yum.repos.d scp /etc/yum.repos.d/CentOS-Base.repo root@nn3:/etc/yum.repos.d scp /etc/yum.repos.d/CentOS-Base.repo root@s1:/etc/yum.repos.d scp /etc/yum.repos.d/CentOS-Base.repo root@s2:/etc/yum.repos.d scp /etc/yum.repos.d/CentOS-Base.repo root@s3:/etc/yum.repos.d
\n执行yum源更新命令1 2 3 yum clean all yum makecache yum update -y
\n安装常用软件 1 yum install -y openssh-server vim gcc gcc-c++ glibc-headers bzip2-devel lzo-devel curl wget openssh-clients zlib-devel autoconf automake cmake libtool openssl-devel fuse-devel snappy-devel telnet unzip zip net-tools.x86_64 firewalld systemd ntp unrar bzip2
\n \nJDK安装 \n注意需要在六台机器依次执行
\n \n\n上传到/tmp目录下，安装 1 2 cd /tmprpm -ivh jdk-8u144-linux-x64.rpm
\n配置环境变量 1 2 3 4 5 ln -s /usr/java/jdk1.8.0_144/ /usr/java/jdk1.8echo 'export JAVA_HOME=/usr/java/jdk1.8' >> /etc/profile.d/myEnv.sh echo 'export PATH=$PATH:$JAVA_HOME/bin' >> /etc/profile.d/myEnv.sh source /etc/profile java -version
\n \n修改主机名和主机名映射 6台机器分别为nn1、nn2、nn3、s1、s2、s3
\n修改为
\n1 2 3 4 5 6 192.168.1.30 nn1 192.168.1.31 nn2 192.168.1.32 nn3 192.168.1.33 s1 192.168.1.34 s2 192.168.1.35 s3
创建hadoop用户 1 2 3 4 useradd hadoop passwd hadoop
禁止非 wheel 组用户切换到root，配置免密切换root \n修改/etc/pam.d/su配置 1 2 sed -i 's/#auth\\t\\trequired\\tpam_wheel.so/auth\\t\\trequired\\tpam_wheel.so/g' '/etc/pam.d/su' sed -i 's/#auth\\t\\tsufficient\\tpam_wheel.so/auth\\t\\tsufficient\\tpam_wheel.so/g' '/etc/pam.d/su'
\n修改/etc/login.defs文件 1 echo "SU_WHEEL_ONLY yes" >> /etc/login.defs
\n添加用户到管理员，禁止普通用户su 到 root 1 2 3 4 gpasswd -a hadoop wheel cat /etc/group | grep wheel
\n \n给hadoop用户，配置SSH密钥 配置hadoop用户ssh免密码登录到hadoop \n仅在nn1执行这段脚本命令即可su - hadoop , mkdir ~/.ssh 需要在其他主机执行一下 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 su - hadoop ssh-keygen -t rsa -f ~/.ssh/id_rsa -P '' ssh-copy-id nn1 ssh-copy-id nn2 ssh-copy-id nn3 ssh-copy-id s1 ssh-copy-id s2 ssh-copy-id s3 scp /home/hadoop/.ssh/id_rsa hadoop@nn2:/home/hadoop/.ssh scp /home/hadoop/.ssh/id_rsa hadoop@nn3:/home/hadoop/.ssh scp /home/hadoop/.ssh/id_rsa hadoop@s1:/home/hadoop/.ssh scp /home/hadoop/.ssh/id_rsa hadoop@s2:/home/hadoop/.ssh scp /home/hadoop/.ssh/id_rsa hadoop@s3:/home/hadoop/.ssh
\n \n配置hadoop用户ssh免密码登录到root \n同上 1 2 3 4 5 6 7 8 9 10 11 ssh-copy-id root@nn1 ssh-copy-id root@ nn2 ssh-copy-id root@nn3 ssh-copy-id root@s1 ssh-copy-id root@s2 ssh-copy-id root@s3 scp /home/hadoop/.ssh/id_rsa root@nn2:/root/.ssh scp /home/hadoop/.ssh/id_rsa root@nn3:/root/.ssh scp /home/hadoop/.ssh/id_rsa root@s1:/root/.ssh scp /home/hadoop/.ssh/id_rsa root@s2:/root/.ssh scp /home/hadoop/.ssh/id_rsa root@s3:/root/.ssh
\n \n脚本配置 \n","cover":false,"excerpt":"","more":"防火墙关闭 1 2 3 systemctl stop firewalld systemctl disable firewalld
配置yum源 \n下载 repo 文件：Centos-7.repo /tmp，进入到/tmp \n备份并且替换系统的repo文件1 2 3 4 \tcp Centos-7.repo /etc/yum.repos.d/ cd /etc/yum.repos.d/ mv CentOS-Base.repo CentOS-Base.repo.bak mv Centos-7.repo CentOS-Base.repo
\n将nn1上的CentOS-Base.repo拷贝到其他主机1 2 3 4 5 scp /etc/yum.repos.d/CentOS-Base.repo root@nn2:/etc/yum.repos.d scp /etc/yum.repos.d/CentOS-Base.repo root@nn3:/etc/yum.repos.d scp /etc/yum.repos.d/CentOS-Base.repo root@s1:/etc/yum.repos.d scp /etc/yum.repos.d/CentOS-Base.repo root@s2:/etc/yum.repos.d scp /etc/yum.repos.d/CentOS-Base.repo root@s3:/etc/yum.repos.d
\n执行yum源更新命令1 2 3 yum clean all yum makecache yum update -y
\n安装常用软件 1 yum install -y openssh-server vim gcc gcc-c++ glibc-headers bzip2-devel lzo-devel curl wget openssh-clients zlib-devel autoconf automake cmake libtool openssl-devel fuse-devel snappy-devel telnet unzip zip net-tools.x86_64 firewalld systemd ntp unrar bzip2
\n \nJDK安装 \n注意需要在六台机器依次执行
\n \n\n上传到/tmp目录下，安装 1 2 cd /tmprpm -ivh jdk-8u144-linux-x64.rpm
\n配置环境变量 1 2 3 4 5 ln -s /usr/java/jdk1.8.0_144/ /usr/java/jdk1.8echo 'export JAVA_HOME=/usr/java/jdk1.8' >> /etc/profile.d/myEnv.sh echo 'export PATH=$PATH:$JAVA_HOME/bin' >> /etc/profile.d/myEnv.sh source /etc/profile java -version
\n \n修改主机名和主机名映射 6台机器分别为nn1、nn2、nn3、s1、s2、s3
\n修改为
\n1 2 3 4 5 6 192.168.1.30 nn1 192.168.1.31 nn2 192.168.1.32 nn3 192.168.1.33 s1 192.168.1.34 s2 192.168.1.35 s3
创建hadoop用户 1 2 3 4 useradd hadoop passwd hadoop
禁止非 wheel 组用户切换到root，配置免密切换root \n修改/etc/pam.d/su配置 1 2 sed -i 's/#auth\\t\\trequired\\tpam_wheel.so/auth\\t\\trequired\\tpam_wheel.so/g' '/etc/pam.d/su' sed -i 's/#auth\\t\\tsufficient\\tpam_wheel.so/auth\\t\\tsufficient\\tpam_wheel.so/g' '/etc/pam.d/su'
\n修改/etc/login.defs文件 1 echo "SU_WHEEL_ONLY yes" >> /etc/login.defs
\n添加用户到管理员，禁止普通用户su 到 root 1 2 3 4 gpasswd -a hadoop wheel cat /etc/group | grep wheel
\n \n给hadoop用户，配置SSH密钥 配置hadoop用户ssh免密码登录到hadoop \n仅在nn1执行这段脚本命令即可su - hadoop , mkdir ~/.ssh 需要在其他主机执行一下 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 su - hadoop ssh-keygen -t rsa -f ~/.ssh/id_rsa -P '' ssh-copy-id nn1 ssh-copy-id nn2 ssh-copy-id nn3 ssh-copy-id s1 ssh-copy-id s2 ssh-copy-id s3 scp /home/hadoop/.ssh/id_rsa hadoop@nn2:/home/hadoop/.ssh scp /home/hadoop/.ssh/id_rsa hadoop@nn3:/home/hadoop/.ssh scp /home/hadoop/.ssh/id_rsa hadoop@s1:/home/hadoop/.ssh scp /home/hadoop/.ssh/id_rsa hadoop@s2:/home/hadoop/.ssh scp /home/hadoop/.ssh/id_rsa hadoop@s3:/home/hadoop/.ssh
\n \n配置hadoop用户ssh免密码登录到root \n同上 1 2 3 4 5 6 7 8 9 10 11 ssh-copy-id root@nn1 ssh-copy-id root@ nn2 ssh-copy-id root@nn3 ssh-copy-id root@s1 ssh-copy-id root@s2 ssh-copy-id root@s3 scp /home/hadoop/.ssh/id_rsa root@nn2:/root/.ssh scp /home/hadoop/.ssh/id_rsa root@nn3:/root/.ssh scp /home/hadoop/.ssh/id_rsa root@s1:/root/.ssh scp /home/hadoop/.ssh/id_rsa root@s2:/root/.ssh scp /home/hadoop/.ssh/id_rsa root@s3:/root/.ssh
\n \n脚本配置 \n"},{"title":"Hadoop集群HDFS配置","top_img":"/img/site01.jpg","top_img_height":"800px","abbrlink":61252,"date":"2024-09-11T14:45:40.000Z","_content":"\n### 上传`hadoop-3.1.4.tar.gz`到`/tmp`，解压\n>注意在六台机器均上传到`/tmp`\n```bash\n# 在6台机器执行\nsudo tar -zxvf /tmp/hadoop-3.1.4.tar.gz -C /usr/local/\n# 分发到其他主机\nssh_root.sh chown -R hadoop:hadoop /usr/local/hadoop-3.1.4\nssh_root.sh ln -s /usr/local/hadoop-3.1.4/ /usr/local/hadoop\n```\n### 配置环境变量\n```bash\necho 'export HADOOP_HOME=/usr/local/hadoop' >> /etc/profile.d/myEnv.sh\necho 'export PATH=$PATH:$HADOOP_HOME/bin' >> /etc/profile.d/myEnv.sh\necho 'export PATH=$PATH:$HADOOP_HOME/sbin' >> /etc/profile.d/myEnv.sh\n```\n\n```bash\n# 分发到nn2、nn3、s1、s2、s3\nscp_all.sh /etc/profile.d/myEnv.sh /etc/profile.d/\n# source 环境变量\nssh_root.sh source /etc/profile\n```\n>还需要创建 `/data`这个目录，由于nn1、nn2、nn3已经创建`/data`，其他三台需要创建一下\n```bash\n### 在s1、s2、s3执行\nsudo mkdir /data\nsudo chown -R hadoop:hadoop /data\n```\n\n### 修改core-site.xml\n```bash\nvim /usr/local/hadoop/etc/hadoop/core-site.xml \n```\n\n```xml\n\n\t\n\t fs.defaultFS \n\t hdfs://ns1 \n\t 默认文件服务的协议和NS逻辑名称，和hdfs-site.xml里的对应此配置替代了1.0里的fs.default.name \n\t \n\t\n\t\n\t hadoop.tmp.dir \n\t /data/tmp \n\t 数据存储目录 \n\t \n\t\n\t\n\t hadoop.proxyuser.root.groups \n\t hadoop \n\t \n\t hdfs dfsadmin –refreshSuperUserGroupsConfiguration,\n\t yarn rmadmin –refreshSuperUserGroupsConfiguration\n\t 使用这两个命令不用重启就能刷新\n\t \n\t \n\t\n\t\n\t hadoop.proxyuser.root.hosts \n\t localhost \n\t 本地代理 \n\t \n\t\n\t\n\t \n\t\tha.zookeeper.quorum \n\t\tnn1:2181,nn2:2181,nn3:2181 \n\t\tHA使用的zookeeper地址 \n\t \n \n```\n### 修改`hdfs-site.xml`\n```bash\nvim /usr/local/hadoop/etc/hadoop/hdfs-site.xml \n```\n\n```xml\n\n \n dfs.namenode.name.dir \n /data/namenode \n namenode本地文件存放地址 \n \n \n \n dfs.nameservices \n ns1 \n 提供服务的NS逻辑名称，与core-site.xml里的对应 \n \n \n \n \n \n dfs.ha.namenodes.ns1 \n nn1,nn2,nn3 \n 列出该逻辑名称下的NameNode逻辑名称 \n \n \n \n dfs.namenode.rpc-address.ns1.nn1 \n nn1:9000 \n 指定NameNode的RPC位置 \n \n \n \n dfs.namenode.http-address.ns1.nn1 \n nn1:50070 \n 指定NameNode的Web Server位置 \n \n \n \n dfs.namenode.rpc-address.ns1.nn2 \n nn2:9000 \n 指定NameNode的RPC位置 \n \n \n \n dfs.namenode.http-address.ns1.nn2 \n nn2:50070 \n 指定NameNode的Web Server位置 \n \n \n \n dfs.namenode.rpc-address.ns1.nn3 \n nn3:9000 \n 指定NameNode的RPC位置 \n \n \n \n dfs.namenode.http-address.ns1.nn3 \n nn3:50070 \n 指定NameNode的Web Server位置 \n \n \n \n dfs.namenode.handler.count \n 77 \n namenode的工作线程数 \n \n\n \n \n dfs.namenode.shared.edits.dir \n qjournal://nn1:8485;nn2:8485;nn3:8485/ns1 \n 指定用于HA存放edits的共享存储，通常是namenode的所在机器 \n \n \n \n dfs.journalnode.edits.dir \n /data/journaldata/ \n journaldata服务存放文件的地址 \n \n \n \n ipc.client.connect.max.retries \n 10 \n namenode和journalnode的链接重试次数10次 \n \n \n \n ipc.client.connect.retry.interval \n 10000 \n 重试的间隔时间10s \n \n\n \n \n dfs.ha.fencing.methods \n sshfence \n 指定HA做隔离的方法，缺省是ssh，可设为shell，稍后详述 \n \n \n \n dfs.ha.fencing.ssh.private-key-files \n /home/hadoop/.ssh/id_rsa \n 杀死命令脚本的免密配置秘钥 \n \n \n \n dfs.client.failover.proxy.provider.ns1 \n org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider \n 指定客户端用于HA切换的代理类，不同的NS可以用不同的代理类以上示例为Hadoop 2.0自带的缺省代理类 \n \n \n \n dfs.client.failover.proxy.provider.auto-ha \n org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider \n \n \n \n dfs.ha.automatic-failover.enabled \n true \n \n\t\n\t\n\t dfs.datanode.data.dir \n\t /data/datanode \n\t datanode本地文件存放地址 \n\t \n\t\n\t dfs.replication \n\t 3 \n\t 文件复本数 \n\t \n\t\n\t dfs.namenode.datanode.registration.ip-hostname-check \n\t false \n\t \n\t\n\t dfs.client.use.datanode.hostname \n\t true \n\t \n\t\n\t dfs.datanode.use.datanode.hostname \n\t true \n\t \n \n```\n### 修改`hadoop-env.sh`\n```bash\nvim /usr/local/hadoop/etc/hadoop/hadoop-env.sh\n```\n\n```bash\n# 添加这两行\nsource /etc/profile \nexport HADOOP_HEAPSIZE_MAX=512\n```\n### 分发这些配置文件\n```bash\nscp_all.sh /usr/local/hadoop/etc/hadoop/core-site.xml /usr/local/hadoop/etc/hadoop/\nscp_all.sh /usr/local/hadoop/etc/hadoop/hdfs-site.xml /usr/local/hadoop/etc/hadoop/\nscp_all.sh /usr/local/hadoop/etc/hadoop/hadoop-env.sh /usr/local/hadoop/etc/hadoop/\n```\n### 集群初始化\n- 需要先启动zookeeper集群\n ```bash\n ssh_all_zookeeper.sh /usr/local/zookeeper/bin/zkServer.sh start\n ```\n\n ```bash\n # 第一次启动先启动journalnode，便于3个namenode的元数据同步\n ssh_all_zookeeper.sh hadoop-daemon.sh start journalnode\n ```\n- `zkfc`搭建\n ```bash\n #在nn1节点执行 \n hdfs zkfc -formatZK\n #nn1 nn2 nn3启动zkfc \n hadoop-daemon.sh start zkfc\n ```\n- 初始化nn1的namenode，在nn1执行\n ```bash\n hdfs namenode -format \n hadoop-daemon.sh start namenode\n ```\n- 格式化第二台和第三台namenode，并且启动namenode,在nn2、nn3执行\n ```bash\n hdfs namenode -bootstrapStandby \n hadoop-daemon.sh start namenode\n ```\n- 修改**workers**\n ```bash\n vim /usr/local/hadoop/etc/hadoop/workers\n ```\n 修改为\n ```text\n s1\n s2\n s3\n ```\n 分发给其他机器\n ```bash\n scp_all.sh /usr/local/hadoop/etc/hadoop/workers /usr/local/hadoop/etc/hadoop\n ```\n- 启动datanode节点，在s1、s2、s3执行\n\n ```bash\n #启动各个节点的datanode\n hadoop-daemons.sh start datanode\n ```\n### 集群启动\n```bash\nstart-dfs.sh\n```","source":"_posts/bigdata/hadoop/hdfs.md","raw":"---\ntitle: Hadoop集群HDFS配置\ntop_img: /img/site01.jpg\ntop_img_height: 800px\nabbrlink: 61252\ndate: 2024-09-011 22:45:40\n---\n\n### 上传`hadoop-3.1.4.tar.gz`到`/tmp`，解压\n>注意在六台机器均上传到`/tmp`\n```bash\n# 在6台机器执行\nsudo tar -zxvf /tmp/hadoop-3.1.4.tar.gz -C /usr/local/\n# 分发到其他主机\nssh_root.sh chown -R hadoop:hadoop /usr/local/hadoop-3.1.4\nssh_root.sh ln -s /usr/local/hadoop-3.1.4/ /usr/local/hadoop\n```\n### 配置环境变量\n```bash\necho 'export HADOOP_HOME=/usr/local/hadoop' >> /etc/profile.d/myEnv.sh\necho 'export PATH=$PATH:$HADOOP_HOME/bin' >> /etc/profile.d/myEnv.sh\necho 'export PATH=$PATH:$HADOOP_HOME/sbin' >> /etc/profile.d/myEnv.sh\n```\n\n```bash\n# 分发到nn2、nn3、s1、s2、s3\nscp_all.sh /etc/profile.d/myEnv.sh /etc/profile.d/\n# source 环境变量\nssh_root.sh source /etc/profile\n```\n>还需要创建 `/data`这个目录，由于nn1、nn2、nn3已经创建`/data`，其他三台需要创建一下\n```bash\n### 在s1、s2、s3执行\nsudo mkdir /data\nsudo chown -R hadoop:hadoop /data\n```\n\n### 修改core-site.xml\n```bash\nvim /usr/local/hadoop/etc/hadoop/core-site.xml \n```\n\n```xml\n\n\t\n\t fs.defaultFS \n\t hdfs://ns1 \n\t 默认文件服务的协议和NS逻辑名称，和hdfs-site.xml里的对应此配置替代了1.0里的fs.default.name \n\t \n\t\n\t\n\t hadoop.tmp.dir \n\t /data/tmp \n\t 数据存储目录 \n\t \n\t\n\t\n\t hadoop.proxyuser.root.groups \n\t hadoop \n\t \n\t hdfs dfsadmin –refreshSuperUserGroupsConfiguration,\n\t yarn rmadmin –refreshSuperUserGroupsConfiguration\n\t 使用这两个命令不用重启就能刷新\n\t \n\t \n\t\n\t\n\t hadoop.proxyuser.root.hosts \n\t localhost \n\t 本地代理 \n\t \n\t\n\t\n\t \n\t\tha.zookeeper.quorum \n\t\tnn1:2181,nn2:2181,nn3:2181 \n\t\tHA使用的zookeeper地址 \n\t \n \n```\n### 修改`hdfs-site.xml`\n```bash\nvim /usr/local/hadoop/etc/hadoop/hdfs-site.xml \n```\n\n```xml\n\n \n dfs.namenode.name.dir \n /data/namenode \n namenode本地文件存放地址 \n \n \n \n dfs.nameservices \n ns1 \n 提供服务的NS逻辑名称，与core-site.xml里的对应 \n \n \n \n \n \n dfs.ha.namenodes.ns1 \n nn1,nn2,nn3 \n 列出该逻辑名称下的NameNode逻辑名称 \n \n \n \n dfs.namenode.rpc-address.ns1.nn1 \n nn1:9000 \n 指定NameNode的RPC位置 \n \n \n \n dfs.namenode.http-address.ns1.nn1 \n nn1:50070 \n 指定NameNode的Web Server位置 \n \n \n \n dfs.namenode.rpc-address.ns1.nn2 \n nn2:9000 \n 指定NameNode的RPC位置 \n \n \n \n dfs.namenode.http-address.ns1.nn2 \n nn2:50070 \n 指定NameNode的Web Server位置 \n \n \n \n dfs.namenode.rpc-address.ns1.nn3 \n nn3:9000 \n 指定NameNode的RPC位置 \n \n \n \n dfs.namenode.http-address.ns1.nn3 \n nn3:50070 \n 指定NameNode的Web Server位置 \n \n \n \n dfs.namenode.handler.count \n 77 \n namenode的工作线程数 \n \n\n \n \n dfs.namenode.shared.edits.dir \n qjournal://nn1:8485;nn2:8485;nn3:8485/ns1 \n 指定用于HA存放edits的共享存储，通常是namenode的所在机器 \n \n \n \n dfs.journalnode.edits.dir \n /data/journaldata/ \n journaldata服务存放文件的地址 \n \n \n \n ipc.client.connect.max.retries \n 10 \n namenode和journalnode的链接重试次数10次 \n \n \n \n ipc.client.connect.retry.interval \n 10000 \n 重试的间隔时间10s \n \n\n \n \n dfs.ha.fencing.methods \n sshfence \n 指定HA做隔离的方法，缺省是ssh，可设为shell，稍后详述 \n \n \n \n dfs.ha.fencing.ssh.private-key-files \n /home/hadoop/.ssh/id_rsa \n 杀死命令脚本的免密配置秘钥 \n \n \n \n dfs.client.failover.proxy.provider.ns1 \n org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider \n 指定客户端用于HA切换的代理类，不同的NS可以用不同的代理类以上示例为Hadoop 2.0自带的缺省代理类 \n \n \n \n dfs.client.failover.proxy.provider.auto-ha \n org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider \n \n \n \n dfs.ha.automatic-failover.enabled \n true \n \n\t\n\t\n\t dfs.datanode.data.dir \n\t /data/datanode \n\t datanode本地文件存放地址 \n\t \n\t\n\t dfs.replication \n\t 3 \n\t 文件复本数 \n\t \n\t\n\t dfs.namenode.datanode.registration.ip-hostname-check \n\t false \n\t \n\t\n\t dfs.client.use.datanode.hostname \n\t true \n\t \n\t\n\t dfs.datanode.use.datanode.hostname \n\t true \n\t \n \n```\n### 修改`hadoop-env.sh`\n```bash\nvim /usr/local/hadoop/etc/hadoop/hadoop-env.sh\n```\n\n```bash\n# 添加这两行\nsource /etc/profile \nexport HADOOP_HEAPSIZE_MAX=512\n```\n### 分发这些配置文件\n```bash\nscp_all.sh /usr/local/hadoop/etc/hadoop/core-site.xml /usr/local/hadoop/etc/hadoop/\nscp_all.sh /usr/local/hadoop/etc/hadoop/hdfs-site.xml /usr/local/hadoop/etc/hadoop/\nscp_all.sh /usr/local/hadoop/etc/hadoop/hadoop-env.sh /usr/local/hadoop/etc/hadoop/\n```\n### 集群初始化\n- 需要先启动zookeeper集群\n ```bash\n ssh_all_zookeeper.sh /usr/local/zookeeper/bin/zkServer.sh start\n ```\n\n ```bash\n # 第一次启动先启动journalnode，便于3个namenode的元数据同步\n ssh_all_zookeeper.sh hadoop-daemon.sh start journalnode\n ```\n- `zkfc`搭建\n ```bash\n #在nn1节点执行 \n hdfs zkfc -formatZK\n #nn1 nn2 nn3启动zkfc \n hadoop-daemon.sh start zkfc\n ```\n- 初始化nn1的namenode，在nn1执行\n ```bash\n hdfs namenode -format \n hadoop-daemon.sh start namenode\n ```\n- 格式化第二台和第三台namenode，并且启动namenode,在nn2、nn3执行\n ```bash\n hdfs namenode -bootstrapStandby \n hadoop-daemon.sh start namenode\n ```\n- 修改**workers**\n ```bash\n vim /usr/local/hadoop/etc/hadoop/workers\n ```\n 修改为\n ```text\n s1\n s2\n s3\n ```\n 分发给其他机器\n ```bash\n scp_all.sh /usr/local/hadoop/etc/hadoop/workers /usr/local/hadoop/etc/hadoop\n ```\n- 启动datanode节点，在s1、s2、s3执行\n\n ```bash\n #启动各个节点的datanode\n hadoop-daemons.sh start datanode\n ```\n### 集群启动\n```bash\nstart-dfs.sh\n```","slug":"bigdata/hadoop/hdfs","published":1,"updated":"2024-09-11T14:51:42.712Z","comments":1,"layout":"post","photos":[],"_id":"cmgsv0jnq00133wahetp950z2","content":"上传hadoop-3.1.4.tar.gz到/tmp，解压 \n注意在六台机器均上传到/tmp
\n \n1 2 3 4 5 sudo tar -zxvf /tmp/hadoop-3.1.4.tar.gz -C /usr/local/ssh_root.sh chown -R hadoop:hadoop /usr/local/hadoop-3.1.4 ssh_root.sh ln -s /usr/local/hadoop-3.1.4/ /usr/local/hadoop
配置环境变量 1 2 3 echo 'export HADOOP_HOME=/usr/local/hadoop' >> /etc/profile.d/myEnv.shecho 'export PATH=$PATH:$HADOOP_HOME/bin' >> /etc/profile.d/myEnv.shecho 'export PATH=$PATH:$HADOOP_HOME/sbin' >> /etc/profile.d/myEnv.sh
1 2 3 4 scp_all.sh /etc/profile.d/myEnv.sh /etc/profile.d/ ssh_root.sh source /etc/profile
\n还需要创建 /data这个目录，由于nn1、nn2、nn3已经创建/data，其他三台需要创建一下
\n \n1 2 3 sudo mkdir /datasudo chown -R hadoop:hadoop /data
修改core-site.xml 1 vim /usr/local/hadoop/etc/hadoop/core-site.xml
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 <configuration > \t<property > \t <name > fs.defaultFS</name > \t <value > hdfs://ns1</value > \t <description > 默认文件服务的协议和NS逻辑名称，和hdfs-site.xml里的对应此配置替代了1.0里的fs.default.name</description > \t</property > \t \t<property > \t <name > hadoop.tmp.dir</name > \t <value > /data/tmp</value > \t <description > 数据存储目录</description > \t</property > \t \t<property > \t <name > hadoop.proxyuser.root.groups</name > \t <value > hadoop</value > \t <description > \t hdfs dfsadmin –refreshSuperUserGroupsConfiguration, \t yarn rmadmin –refreshSuperUserGroupsConfiguration \t 使用这两个命令不用重启就能刷新 \t </description > \t</property > \t \t<property > \t <name > hadoop.proxyuser.root.hosts</name > \t <value > localhost</value > \t <description > 本地代理</description > \t</property > \t \t \t<property > \t\t<name > ha.zookeeper.quorum</name > \t\t<value > nn1:2181,nn2:2181,nn3:2181</value > \t\t<description > HA使用的zookeeper地址</description > \t</property > </configuration >
修改hdfs-site.xml 1 vim /usr/local/hadoop/etc/hadoop/hdfs-site.xml
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 <configuration > <property > <name > dfs.namenode.name.dir</name > <value > /data/namenode</value > <description > namenode本地文件存放地址</description > </property > <property > <name > dfs.nameservices</name > <value > ns1</value > <description > 提供服务的NS逻辑名称，与core-site.xml里的对应</description > </property > <property > <name > dfs.ha.namenodes.ns1</name > <value > nn1,nn2,nn3</value > <description > 列出该逻辑名称下的NameNode逻辑名称</description > </property > <property > <name > dfs.namenode.rpc-address.ns1.nn1</name > <value > nn1:9000</value > <description > 指定NameNode的RPC位置</description > </property > <property > <name > dfs.namenode.http-address.ns1.nn1</name > <value > nn1:50070</value > <description > 指定NameNode的Web Server位置</description > </property > <property > <name > dfs.namenode.rpc-address.ns1.nn2</name > <value > nn2:9000</value > <description > 指定NameNode的RPC位置</description > </property > <property > <name > dfs.namenode.http-address.ns1.nn2</name > <value > nn2:50070</value > <description > 指定NameNode的Web Server位置</description > </property > <property > <name > dfs.namenode.rpc-address.ns1.nn3</name > <value > nn3:9000</value > <description > 指定NameNode的RPC位置</description > </property > <property > <name > dfs.namenode.http-address.ns1.nn3</name > <value > nn3:50070</value > <description > 指定NameNode的Web Server位置</description > </property > <property > <name > dfs.namenode.handler.count</name > <value > 77</value > <description > namenode的工作线程数</description > </property > <property > <name > dfs.namenode.shared.edits.dir</name > <value > qjournal://nn1:8485;nn2:8485;nn3:8485/ns1</value > <description > 指定用于HA存放edits的共享存储，通常是namenode的所在机器</description > </property > <property > <name > dfs.journalnode.edits.dir</name > <value > /data/journaldata/</value > <description > journaldata服务存放文件的地址</description > </property > <property > <name > ipc.client.connect.max.retries</name > <value > 10</value > <description > namenode和journalnode的链接重试次数10次</description > </property > <property > <name > ipc.client.connect.retry.interval</name > <value > 10000</value > <description > 重试的间隔时间10s</description > </property > <property > <name > dfs.ha.fencing.methods</name > <value > sshfence</value > <description > 指定HA做隔离的方法，缺省是ssh，可设为shell，稍后详述</description > </property > <property > <name > dfs.ha.fencing.ssh.private-key-files</name > <value > /home/hadoop/.ssh/id_rsa</value > <description > 杀死命令脚本的免密配置秘钥</description > </property > <property > <name > dfs.client.failover.proxy.provider.ns1</name > <value > org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value > <description > 指定客户端用于HA切换的代理类，不同的NS可以用不同的代理类以上示例为Hadoop 2.0自带的缺省代理类</description > </property > <property > <name > dfs.client.failover.proxy.provider.auto-ha</name > <value > org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value > </property > <property > <name > dfs.ha.automatic-failover.enabled</name > <value > true</value > </property > \t \t<property > \t <name > dfs.datanode.data.dir</name > \t <value > /data/datanode</value > \t <description > datanode本地文件存放地址</description > \t</property > \t<property > \t <name > dfs.replication</name > \t <value > 3</value > \t <description > 文件复本数</description > \t</property > \t<property > \t <name > dfs.namenode.datanode.registration.ip-hostname-check</name > \t <value > false</value > \t</property > \t<property > \t <name > dfs.client.use.datanode.hostname</name > \t <value > true</value > \t</property > \t<property > \t <name > dfs.datanode.use.datanode.hostname</name > \t <value > true</value > \t</property > </configuration >
修改hadoop-env.sh 1 vim /usr/local/hadoop/etc/hadoop/hadoop-env.sh
1 2 3 source /etc/profile export HADOOP_HEAPSIZE_MAX=512
分发这些配置文件 1 2 3 scp_all.sh /usr/local/hadoop/etc/hadoop/core-site.xml /usr/local/hadoop/etc/hadoop/ scp_all.sh /usr/local/hadoop/etc/hadoop/hdfs-site.xml /usr/local/hadoop/etc/hadoop/ scp_all.sh /usr/local/hadoop/etc/hadoop/hadoop-env.sh /usr/local/hadoop/etc/hadoop/
集群初始化 \n集群启动 上传hadoop-3.1.4.tar.gz到/tmp，解压 \n注意在六台机器均上传到/tmp
\n \n1 2 3 4 5 sudo tar -zxvf /tmp/hadoop-3.1.4.tar.gz -C /usr/local/ssh_root.sh chown -R hadoop:hadoop /usr/local/hadoop-3.1.4 ssh_root.sh ln -s /usr/local/hadoop-3.1.4/ /usr/local/hadoop
配置环境变量 1 2 3 echo 'export HADOOP_HOME=/usr/local/hadoop' >> /etc/profile.d/myEnv.shecho 'export PATH=$PATH:$HADOOP_HOME/bin' >> /etc/profile.d/myEnv.shecho 'export PATH=$PATH:$HADOOP_HOME/sbin' >> /etc/profile.d/myEnv.sh
1 2 3 4 scp_all.sh /etc/profile.d/myEnv.sh /etc/profile.d/ ssh_root.sh source /etc/profile
\n还需要创建 /data这个目录，由于nn1、nn2、nn3已经创建/data，其他三台需要创建一下
\n \n1 2 3 sudo mkdir /datasudo chown -R hadoop:hadoop /data
修改core-site.xml 1 vim /usr/local/hadoop/etc/hadoop/core-site.xml
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 <configuration > \t<property > \t <name > fs.defaultFS</name > \t <value > hdfs://ns1</value > \t <description > 默认文件服务的协议和NS逻辑名称，和hdfs-site.xml里的对应此配置替代了1.0里的fs.default.name</description > \t</property > \t \t<property > \t <name > hadoop.tmp.dir</name > \t <value > /data/tmp</value > \t <description > 数据存储目录</description > \t</property > \t \t<property > \t <name > hadoop.proxyuser.root.groups</name > \t <value > hadoop</value > \t <description > \t hdfs dfsadmin –refreshSuperUserGroupsConfiguration, \t yarn rmadmin –refreshSuperUserGroupsConfiguration \t 使用这两个命令不用重启就能刷新 \t </description > \t</property > \t \t<property > \t <name > hadoop.proxyuser.root.hosts</name > \t <value > localhost</value > \t <description > 本地代理</description > \t</property > \t \t \t<property > \t\t<name > ha.zookeeper.quorum</name > \t\t<value > nn1:2181,nn2:2181,nn3:2181</value > \t\t<description > HA使用的zookeeper地址</description > \t</property > </configuration >
修改hdfs-site.xml 1 vim /usr/local/hadoop/etc/hadoop/hdfs-site.xml
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 <configuration > <property > <name > dfs.namenode.name.dir</name > <value > /data/namenode</value > <description > namenode本地文件存放地址</description > </property > <property > <name > dfs.nameservices</name > <value > ns1</value > <description > 提供服务的NS逻辑名称，与core-site.xml里的对应</description > </property > <property > <name > dfs.ha.namenodes.ns1</name > <value > nn1,nn2,nn3</value > <description > 列出该逻辑名称下的NameNode逻辑名称</description > </property > <property > <name > dfs.namenode.rpc-address.ns1.nn1</name > <value > nn1:9000</value > <description > 指定NameNode的RPC位置</description > </property > <property > <name > dfs.namenode.http-address.ns1.nn1</name > <value > nn1:50070</value > <description > 指定NameNode的Web Server位置</description > </property > <property > <name > dfs.namenode.rpc-address.ns1.nn2</name > <value > nn2:9000</value > <description > 指定NameNode的RPC位置</description > </property > <property > <name > dfs.namenode.http-address.ns1.nn2</name > <value > nn2:50070</value > <description > 指定NameNode的Web Server位置</description > </property > <property > <name > dfs.namenode.rpc-address.ns1.nn3</name > <value > nn3:9000</value > <description > 指定NameNode的RPC位置</description > </property > <property > <name > dfs.namenode.http-address.ns1.nn3</name > <value > nn3:50070</value > <description > 指定NameNode的Web Server位置</description > </property > <property > <name > dfs.namenode.handler.count</name > <value > 77</value > <description > namenode的工作线程数</description > </property > <property > <name > dfs.namenode.shared.edits.dir</name > <value > qjournal://nn1:8485;nn2:8485;nn3:8485/ns1</value > <description > 指定用于HA存放edits的共享存储，通常是namenode的所在机器</description > </property > <property > <name > dfs.journalnode.edits.dir</name > <value > /data/journaldata/</value > <description > journaldata服务存放文件的地址</description > </property > <property > <name > ipc.client.connect.max.retries</name > <value > 10</value > <description > namenode和journalnode的链接重试次数10次</description > </property > <property > <name > ipc.client.connect.retry.interval</name > <value > 10000</value > <description > 重试的间隔时间10s</description > </property > <property > <name > dfs.ha.fencing.methods</name > <value > sshfence</value > <description > 指定HA做隔离的方法，缺省是ssh，可设为shell，稍后详述</description > </property > <property > <name > dfs.ha.fencing.ssh.private-key-files</name > <value > /home/hadoop/.ssh/id_rsa</value > <description > 杀死命令脚本的免密配置秘钥</description > </property > <property > <name > dfs.client.failover.proxy.provider.ns1</name > <value > org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value > <description > 指定客户端用于HA切换的代理类，不同的NS可以用不同的代理类以上示例为Hadoop 2.0自带的缺省代理类</description > </property > <property > <name > dfs.client.failover.proxy.provider.auto-ha</name > <value > org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value > </property > <property > <name > dfs.ha.automatic-failover.enabled</name > <value > true</value > </property > \t \t<property > \t <name > dfs.datanode.data.dir</name > \t <value > /data/datanode</value > \t <description > datanode本地文件存放地址</description > \t</property > \t<property > \t <name > dfs.replication</name > \t <value > 3</value > \t <description > 文件复本数</description > \t</property > \t<property > \t <name > dfs.namenode.datanode.registration.ip-hostname-check</name > \t <value > false</value > \t</property > \t<property > \t <name > dfs.client.use.datanode.hostname</name > \t <value > true</value > \t</property > \t<property > \t <name > dfs.datanode.use.datanode.hostname</name > \t <value > true</value > \t</property > </configuration >
修改hadoop-env.sh 1 vim /usr/local/hadoop/etc/hadoop/hadoop-env.sh
1 2 3 source /etc/profile export HADOOP_HEAPSIZE_MAX=512
分发这些配置文件 1 2 3 scp_all.sh /usr/local/hadoop/etc/hadoop/core-site.xml /usr/local/hadoop/etc/hadoop/ scp_all.sh /usr/local/hadoop/etc/hadoop/hdfs-site.xml /usr/local/hadoop/etc/hadoop/ scp_all.sh /usr/local/hadoop/etc/hadoop/hadoop-env.sh /usr/local/hadoop/etc/hadoop/
集群初始化 \n集群启动 Zookeeper脚本配置 \n拷贝ips \n拷贝三个脚本1 2 3 cp scp_all.sh scp_all_zookeeper.sh cp ssh_all.sh ssh_all_zookeeper.sh cp ssh_root.sh ssh_root_zookeeper.sh
\n修改脚本1 2 3 vim scp_all_zookeeper.sh vim ssh_all_zookeeper.sh vim ssh_root_zookeeper.sh
\n \nZookeeper安装 \n上传到/tmp目录下，解压 1 2 3 4 5 sudo tar -zxvf /tmp/zookeeper-3.4.8.tar.gz -C /usr/local/scp -r /usr/local/zookeeper-3.4.8/ root@nn2:/usr/local/ scp -r /usr/local/zookeeper-3.4.8/ root@nn3:/usr/local/ ssh_root_zookeeper.sh chown -R hadoop:hadoop /usr/local/zookeeper-3.4.8 ssh_root_zookeeper.s ln -s /usr/local/zookeeper-3.4.8/ /usr/local/zookeeper
\n \nZookeeper配置 \nZookeeper的命令 1 2 3 4 5 6 7 8 ssh_all_zookeeper.sh /usr/local/zookeeper/bin/zkServer.sh start ssh_all_zookeeper.sh /usr/local/zookeeper/bin/zkServer.sh status ssh_all_zookeeper.sh /usr/local/zookeeper/bin/zkServer.sh stop ssh_all_zookeeper.sh /usr/local/zookeeper/bin/zkServer.sh restart
1 2 3 4 # 启动zookeeper客户端，并连接zookeeper集群 /usr/local/zookeeper/bin/zkCli.sh -server nn1:2181 # 可以简化为: zkCli.sh
Zookeeper脚本配置 \n拷贝ips \n拷贝三个脚本1 2 3 cp scp_all.sh scp_all_zookeeper.sh cp ssh_all.sh ssh_all_zookeeper.sh cp ssh_root.sh ssh_root_zookeeper.sh
\n修改脚本1 2 3 vim scp_all_zookeeper.sh vim ssh_all_zookeeper.sh vim ssh_root_zookeeper.sh
\n \nZookeeper安装 \n上传到/tmp目录下，解压 1 2 3 4 5 sudo tar -zxvf /tmp/zookeeper-3.4.8.tar.gz -C /usr/local/scp -r /usr/local/zookeeper-3.4.8/ root@nn2:/usr/local/ scp -r /usr/local/zookeeper-3.4.8/ root@nn3:/usr/local/ ssh_root_zookeeper.sh chown -R hadoop:hadoop /usr/local/zookeeper-3.4.8 ssh_root_zookeeper.s ln -s /usr/local/zookeeper-3.4.8/ /usr/local/zookeeper
\n \nZookeeper配置 \nZookeeper的命令 1 2 3 4 5 6 7 8 ssh_all_zookeeper.sh /usr/local/zookeeper/bin/zkServer.sh start ssh_all_zookeeper.sh /usr/local/zookeeper/bin/zkServer.sh status ssh_all_zookeeper.sh /usr/local/zookeeper/bin/zkServer.sh stop ssh_all_zookeeper.sh /usr/local/zookeeper/bin/zkServer.sh restart
1 2 3 4 # 启动zookeeper客户端，并连接zookeeper集群 /usr/local/zookeeper/bin/zkCli.sh -server nn1:2181 # 可以简化为: zkCli.sh
由于中国大陆的GFW（防火墙）限制，无法访问外网网络，因此需要访问像GitHub、YouTube这样的VPN服务的可行的方案来保证服务的可靠性。
\n
介绍 \n根据测试，许多提供服务的提供商所在的网站需要使用外部网络才能打开，仅有少部分的网站(比较贵 )可以直接使用
\n \n那么你现在可以有两个选择：
\n\n方案一 ：使用无需外部网络便能开通订阅服务的VPN，费用高，如果你选择此方案，那么你可自行搜索解决，此处仅仅讨论方案二。方案二 ：如果使用此方案，详见下方。 \n解决方案 \n采用方案二 方式
\n这是一些订阅服务推广的链接： https://9.234456.xyz/abc.html?t=638586217737356738 (此链接打开无需使用VPN，但进入对应的机场页面却仍无法打开)
\n此教程中我们使用的机场是
\n\n一元机场: https://xn--4gq62f52gdss.com/ 赔钱机场：https://xn--mes358aby2apfg.com/ \n \n机场选择的建议： \n如何订阅？ 由于需要外部网络才能完成订阅服务的购买，你可以向我的邮箱15202078626@163.com发送你的订阅计划方案，
\n
完成订阅后如何使用？ \n你可以在 Windows、Mac、Android等平台使用此服务https://flowus.cn/shenjian/22f76d4f-e7b3-4b8a-8a89-561566f6eb60
\n \n","cover":false,"excerpt":"","more":"由于中国大陆的GFW（防火墙）限制，无法访问外网网络，因此需要访问像GitHub、YouTube这样的VPN服务的可行的方案来保证服务的可靠性。
\n
介绍 \n根据测试，许多提供服务的提供商所在的网站需要使用外部网络才能打开，仅有少部分的网站(比较贵 )可以直接使用
\n \n那么你现在可以有两个选择：
\n\n方案一 ：使用无需外部网络便能开通订阅服务的VPN，费用高，如果你选择此方案，那么你可自行搜索解决，此处仅仅讨论方案二。方案二 ：如果使用此方案，详见下方。 \n解决方案 \n采用方案二 方式
\n这是一些订阅服务推广的链接： https://9.234456.xyz/abc.html?t=638586217737356738 (此链接打开无需使用VPN，但进入对应的机场页面却仍无法打开)
\n此教程中我们使用的机场是
\n\n一元机场: https://xn--4gq62f52gdss.com/ 赔钱机场：https://xn--mes358aby2apfg.com/ \n \n机场选择的建议： \n如何订阅？ 由于需要外部网络才能完成订阅服务的购买，你可以向我的邮箱15202078626@163.com发送你的订阅计划方案，
\n
完成订阅后如何使用？ \n你可以在 Windows、Mac、Android等平台使用此服务https://flowus.cn/shenjian/22f76d4f-e7b3-4b8a-8a89-561566f6eb60
\n \n"},{"title":"郑伯克段于鄢","abbrlink":58638,"date":"2024-08-09T12:00:13.000Z","_content":"原文如下：\n\n 初，郑武公娶于申【申国】，曰武姜【武为武公谥号，姜为其宗族之性】。生庄公及共叔段【共表示其曾出逃到共，叔为老三，段为名】。庄公寤生【出生时头先出，难产】，惊姜氏，故名曰“寤生”， 遂恶之，爱【喜爱】共叔段，欲立【立为储君】之，亟（qì）【多次】请于武公，及庄公即位，为之【共叔段】请制【一个叫制的封地，虎牢关所在】。公曰：“制，岩邑【险要的城邑】也，虢叔死焉，佗【通“他”，其他】邑唯命（是听）。”请京，使居之，谓之“京城大叔”。","source":"_posts/ancient/guwenguanzhi/1.md","raw":"---\ntitle: 郑伯克段于鄢\ntags: 古文观止\ncategories:\n - 古文观止\nabbrlink: 58638\ndate: 2024-08-09 20:00:13\n---\n原文如下：\n\n 初，郑武公娶于申【申国】，曰武姜【武为武公谥号，姜为其宗族之性】。生庄公及共叔段【共表示其曾出逃到共，叔为老三，段为名】。庄公寤生【出生时头先出，难产】，惊姜氏，故名曰“寤生”， 遂恶之，爱【喜爱】共叔段，欲立【立为储君】之，亟（qì）【多次】请于武公，及庄公即位，为之【共叔段】请制【一个叫制的封地，虎牢关所在】。公曰：“制，岩邑【险要的城邑】也，虢叔死焉，佗【通“他”，其他】邑唯命（是听）。”请京，使居之，谓之“京城大叔”。","slug":"ancient/guwenguanzhi/1","published":1,"updated":"2024-08-10T02:31:03.678Z","comments":1,"layout":"post","photos":[],"_id":"cmgsv0jnr00193wah5y5z0hai","content":"原文如下：
\n 初，郑武公娶于申【申国】，曰武姜【武为武公谥号，姜为其宗族之性】。生庄公及共叔段【共表示其曾出逃到共，叔为老三，段为名】。庄公寤生【出生时头先出，难产】，惊姜氏，故名曰“寤生”， 遂恶之，爱【喜爱】共叔段，欲立【立为储君】之，亟（qì）【多次】请于武公，及庄公即位，为之【共叔段】请制【一个叫制的封地，虎牢关所在】。公曰：“制，岩邑【险要的城邑】也，虢叔死焉，佗【通“他”，其他】邑唯命（是听）。”请京，使居之，谓之“京城大叔”。
\n","cover":false,"excerpt":"","more":"原文如下：
\n 初，郑武公娶于申【申国】，曰武姜【武为武公谥号，姜为其宗族之性】。生庄公及共叔段【共表示其曾出逃到共，叔为老三，段为名】。庄公寤生【出生时头先出，难产】，惊姜氏，故名曰“寤生”， 遂恶之，爱【喜爱】共叔段，欲立【立为储君】之，亟（qì）【多次】请于武公，及庄公即位，为之【共叔段】请制【一个叫制的封地，虎牢关所在】。公曰：“制，岩邑【险要的城邑】也，虢叔死焉，佗【通“他”，其他】邑唯命（是听）。”请京，使居之，谓之“京城大叔”。
\n"},{"title":"Docker被墙，如何继续使用？","top_img":"/img/site01.jpg","top_img_height":"800px","abbrlink":47807,"date":"2024-08-01T01:10:40.000Z","_content":"\n## Docker Download\n> 自从docker官方仓库在中国大陆被墙后，docker的部署方式也发生了改变。\n> 解决docker安装问题：https://github.com/shenjianZ/docker_installer\n\n1. 安装docker \n ```shell\n sudo curl -fsSL https://gitee.com/tech-shrimp/docker_installer/releases/download/latest/linux.sh| bash -s docker --mirror Aliyun\n ```\n \n2. 启动docker\n ```shell\n sudo systemctl start docker\n ```\n \n3. 设置开机自启\n ```shell\n sudo systemctl enable docker\n ```\n \n4. Docker pull images\n > 将image下载到阿里云镜像仓库中\n > 解决docker pull 镜像问题:https://github.com/shenjianZ/docker_image_pusher\n \n 1. **登录阿里云镜像服务** https://cr.console.aliyun.com/,\n\n 启用个人实例，创建一个命名空间（`ALIYUN_NAME_SPACE`）\n\n 2. 在**访问凭证** 可以看到账号 用户名（`ALIYUN_REGISTRY_USER`)\n\n 密码（`ALIYUN_REGISTRY_PASSWORD`)\n\n 仓库地址（`ALIYUN_REGISTRY`）\n\n 3. **启动Action**\n 进入您自己的项目，点击`Action`，启用`Github Action`功能\n \n 4. **配置环境变量**\n 进入Settings->Secret and variables->Actions->New Repository secret\n 将上一步的四个值\n `ALIYUN_NAME_SPACE`,\n \n ` ALIYUN_REGISTRY_USER`，\n \n `ALIYUN_REGISTRY_PASSWORD`，\n \n `ALIYUN_REGISTRY`\n 配置成环境变量\n \n 5. **添加镜像**\n 打开`images.txt`文件，添加你想要的镜像 可以加tag\n \n 6. 使用镜像\n 回到阿里云，镜像仓库，点击任意镜像，可查看镜像状态。(可以改成公开，拉取镜像免登录)","source":"_posts/frontend/deploy/deploy.md","raw":"---\ntitle: Docker被墙，如何继续使用？\ntop_img: /img/site01.jpg\ntop_img_height: 800px\nabbrlink: 47807\ndate: 2024-08-01 09:10:40\ntags:\n---\n\n## Docker Download\n> 自从docker官方仓库在中国大陆被墙后，docker的部署方式也发生了改变。\n> 解决docker安装问题：https://github.com/shenjianZ/docker_installer\n\n1. 安装docker \n ```shell\n sudo curl -fsSL https://gitee.com/tech-shrimp/docker_installer/releases/download/latest/linux.sh| bash -s docker --mirror Aliyun\n ```\n \n2. 启动docker\n ```shell\n sudo systemctl start docker\n ```\n \n3. 设置开机自启\n ```shell\n sudo systemctl enable docker\n ```\n \n4. Docker pull images\n > 将image下载到阿里云镜像仓库中\n > 解决docker pull 镜像问题:https://github.com/shenjianZ/docker_image_pusher\n \n 1. **登录阿里云镜像服务** https://cr.console.aliyun.com/,\n\n 启用个人实例，创建一个命名空间（`ALIYUN_NAME_SPACE`）\n\n 2. 在**访问凭证** 可以看到账号 用户名（`ALIYUN_REGISTRY_USER`)\n\n 密码（`ALIYUN_REGISTRY_PASSWORD`)\n\n 仓库地址（`ALIYUN_REGISTRY`）\n\n 3. **启动Action**\n 进入您自己的项目，点击`Action`，启用`Github Action`功能\n \n 4. **配置环境变量**\n 进入Settings->Secret and variables->Actions->New Repository secret\n 将上一步的四个值\n `ALIYUN_NAME_SPACE`,\n \n ` ALIYUN_REGISTRY_USER`，\n \n `ALIYUN_REGISTRY_PASSWORD`，\n \n `ALIYUN_REGISTRY`\n 配置成环境变量\n \n 5. **添加镜像**\n 打开`images.txt`文件，添加你想要的镜像 可以加tag\n \n 6. 使用镜像\n 回到阿里云，镜像仓库，点击任意镜像，可查看镜像状态。(可以改成公开，拉取镜像免登录)","slug":"frontend/deploy/deploy","published":1,"updated":"2024-08-09T12:21:50.045Z","comments":1,"layout":"post","photos":[],"_id":"cmgsv0jns001b3wah7t2y19hx","content":"Docker Download \n自从docker官方仓库在中国大陆被墙后，docker的部署方式也发生了改变。https://github.com/shenjianZ/docker_installer
\n \n\n安装docker
\n 1 sudo curl -fsSL https://gitee.com/tech-shrimp/docker_installer/releases/download/latest/linux.sh| bash -s docker --mirror Aliyun
启动docker
\n 1 sudo systemctl start docker
设置开机自启
\n 1 sudo systemctl enable docker
Docker pull images
\n\n将image下载到阿里云镜像仓库中https://github.com/shenjianZ/docker_image_pusher
\n \n\n登录阿里云镜像服务 https://cr.console.aliyun.com/ ,
\n启用个人实例，创建一个命名空间（ALIYUN_NAME_SPACE）
\n在访问凭证 可以看到账号 用户名（ALIYUN_REGISTRY_USER)
\n密码（ALIYUN_REGISTRY_PASSWORD)
\n仓库地址（ALIYUN_REGISTRY）
\n启动Action Action，启用Github Action功能
\n配置环境变量 ALIYUN_NAME_SPACE,
\n ALIYUN_REGISTRY_USER，
\n ALIYUN_REGISTRY_PASSWORD，
\n ALIYUN_REGISTRY
\n添加镜像 images.txt文件，添加你想要的镜像 可以加tag
\n使用镜像
\n \n \n","cover":false,"excerpt":"","more":"Docker Download \n自从docker官方仓库在中国大陆被墙后，docker的部署方式也发生了改变。https://github.com/shenjianZ/docker_installer
\n \n\n安装docker
\n 1 sudo curl -fsSL https://gitee.com/tech-shrimp/docker_installer/releases/download/latest/linux.sh| bash -s docker --mirror Aliyun
启动docker
\n 1 sudo systemctl start docker
设置开机自启
\n 1 sudo systemctl enable docker
Docker pull images
\n\n将image下载到阿里云镜像仓库中https://github.com/shenjianZ/docker_image_pusher
\n \n\n登录阿里云镜像服务 https://cr.console.aliyun.com/ ,
\n启用个人实例，创建一个命名空间（ALIYUN_NAME_SPACE）
\n在访问凭证 可以看到账号 用户名（ALIYUN_REGISTRY_USER)
\n密码（ALIYUN_REGISTRY_PASSWORD)
\n仓库地址（ALIYUN_REGISTRY）
\n启动Action Action，启用Github Action功能
\n配置环境变量 ALIYUN_NAME_SPACE,
\n ALIYUN_REGISTRY_USER，
\n ALIYUN_REGISTRY_PASSWORD，
\n ALIYUN_REGISTRY
\n添加镜像 images.txt文件，添加你想要的镜像 可以加tag
\n使用镜像
\n \n \n"}],"PostAsset":[],"PostCategory":[{"post_id":"cmgsv0jnk000g3wah2r7mcppb","category_id":"cmgsv0jnj000e3wahedhm0vqs","_id":"cmgsv0jnm000m3wahd881dlbe"},{"post_id":"cmgsv0jnf000b3wahhxw69j7q","category_id":"cmgsv0jnj000e3wahedhm0vqs","_id":"cmgsv0jnn000p3wahh6yka5b6"},{"post_id":"cmgsv0jnk000j3wah4i4q6sze","category_id":"cmgsv0jnj000e3wahedhm0vqs","_id":"cmgsv0jnn000r3wah6usv5wwn"},{"post_id":"cmgsv0jni000d3wah7wwg0zp1","category_id":"cmgsv0jnj000e3wahedhm0vqs","_id":"cmgsv0jnn000t3wah2eufgco3"},{"post_id":"cmgsv0jnj000f3wah7exd6oi7","category_id":"cmgsv0jnj000e3wahedhm0vqs","_id":"cmgsv0jnn000v3wah5k8s24wg"},{"post_id":"cmgsv0jnr00193wah5y5z0hai","category_id":"cmgsv0jns001c3wahdvavch4e","_id":"cmgsv0jnt001f3wahazr38gm9"}],"PostTag":[{"post_id":"cmgsv0jne00093wahhwfe0rze","tag_id":"cmgsv0jng000c3wah04uo0ly0","_id":"cmgsv0jnk000i3wahaakoh3vr"},{"post_id":"cmgsv0jnf000b3wahhxw69j7q","tag_id":"cmgsv0jnk000h3wahf4t8fikb","_id":"cmgsv0jnm000o3wahc24s0n3o"},{"post_id":"cmgsv0jni000d3wah7wwg0zp1","tag_id":"cmgsv0jnm000l3wahdjzb9rco","_id":"cmgsv0jnn000s3wahf0769lak"},{"post_id":"cmgsv0jnj000f3wah7exd6oi7","tag_id":"cmgsv0jnn000q3wah26zl0nas","_id":"cmgsv0jnn000w3wah9nz27iee"},{"post_id":"cmgsv0jnk000g3wah2r7mcppb","tag_id":"cmgsv0jnn000u3wah5mnw75qh","_id":"cmgsv0jnn000y3wahgn1ve8t2"},{"post_id":"cmgsv0jnk000j3wah4i4q6sze","tag_id":"cmgsv0jnn000x3wah68a82gkb","_id":"cmgsv0jnn000z3wahev4o94c8"},{"post_id":"cmgsv0jnq00143wahef4aabqf","tag_id":"cmgsv0jnp00123wah6qikd29b","_id":"cmgsv0jnr00163wah3wmv2ih3"},{"post_id":"cmgsv0jno00103wahaue5bi4q","tag_id":"cmgsv0jnp00123wah6qikd29b","_id":"cmgsv0jnr00183wah4e0tf2gc"},{"post_id":"cmgsv0jnr00173wah6pwef0gc","tag_id":"cmgsv0jnr001a3wah3ikcey8g","_id":"cmgsv0jnt001e3wah0rkrex0d"},{"post_id":"cmgsv0jnr00193wah5y5z0hai","tag_id":"cmgsv0jns001d3wah27su13dc","_id":"cmgsv0jnt001g3wahazx4d7xb"}],"Tag":[{"name":"C C++","_id":"cmgsv0jng000c3wah04uo0ly0"},{"name":"decisiontree","_id":"cmgsv0jnk000h3wahf4t8fikb"},{"name":"ensemble-learning","_id":"cmgsv0jnm000l3wahdjzb9rco"},{"name":"KNN","_id":"cmgsv0jnn000q3wah26zl0nas"},{"name":"linear-regression","_id":"cmgsv0jnn000u3wah5mnw75qh"},{"name":"logistic-regression","_id":"cmgsv0jnn000x3wah68a82gkb"},{"name":"uniapp","_id":"cmgsv0jnp00123wah6qikd29b"},{"name":"网络代理","_id":"cmgsv0jnr001a3wah3ikcey8g"},{"name":"古文观止","_id":"cmgsv0jns001d3wah27su13dc"}]}} ![](\"/img/language/c-env-conf.png\")
![](\"/img/machinelearning/random-forest.png\")
![](\"/img/machinelearning/ott.png\")
![](\"/img/machinelearning/knn-01.png\")
![](\"/img/machinelearning/cros-valid.png\")
![](\"/img/machinelearning/kfold-skfold.png\")
\n![](\"/img/machinelearning/linear.png\")
![](\"/img/machinelearning/fitting.png\")
![](\"/img/machinelearning/over_random_sampling.png\")
\n![](\"/img/machinelearning/over_smote_sampling.png\")
\n![](\"/img/machinelearning/under_sampling.png\")
\n![\"000001.png\"](\"/../../../img/000001.png\")
![](\"/img/dingyue.png\")
\n\n### 完成订阅后如何使用？\n> 你可以在 `Windows`、`Mac`、`Android`等平台使用此服务\n> 使用订阅的对应链接： https://flowus.cn/shenjian/22f76d4f-e7b3-4b8a-8a89-561566f6eb60\n\n\n ","source":"_posts/net/jichang/jichang.md","raw":"---\ntitle: 无法访问外网？需要订阅代理服务？\ntags: 网络代理\nabbrlink: 14011\ndate: 2024-08-07 10:06:08\n---\n\n{% note info %}\n**由于中国大陆的GFW（防火墙）限制，无法访问外网网络，因此需要访问像GitHub、YouTube这样的\n的网站将被屏蔽拦截，接下来我将给出一种使用`VPN`服务的可行的方案来保证服务的可靠性。**\n{% endnote %}\n\n### 介绍\n> 根据测试，许多提供服务的提供商所在的网站需要使用`外部网络`才能打开，仅有少部分的网站(**比较贵**)可以直接使用\n> 国内网络环境打开直接购买订阅服务。\n>\n\n那么你现在可以有两个选择：\n1. **方案一**：使用无需`外部网络`便能开通订阅服务的VPN，费用高，如果你选择此方案，那么你可自行搜索解决，此处仅仅讨论方案二。\n2. **方案二**：如果使用此方案，详见下方。\n\n\n### 解决方案\n> 采用**方案二**方式\n> \n> 这是一些订阅服务推广的链接： https://9.234456.xyz/abc.html?t=638586217737356738 (此链接打开无需使用VPN，但进入对应的机场页面却仍无法打开)\n> \n> 此教程中我们使用的机场是 \n> 1. `一元机场`: https://xn--4gq62f52gdss.com/\n> 2. `赔钱机场`：https://xn--mes358aby2apfg.com/\n\n### 机场选择的建议：\n\n- `一元机场`\n \n 可以看到\n - `12元/年`,每月50GB的可用额度，下个月重置流量额度\n - `15元/季`，即为`60元/年`，每月有4000GB的可用额度，下个月重置流量额度\n - `7元/月`，即为`84元/年`，每个月8000GB的可用额度，下个月重置流量额度\n 根据我个人的使用情况，大多数情况下我每月的流量使用未超过50GB，如果没有频繁的流量使用，\n 建议选择`12元/年`，否则可以选择`15元/季`，这意味着每月将有4000GB的可用额度\n\n- `赔钱机场`\n\n \n 可以看到\n - `18元/年`,每月100GB的可用额度，允许最多10个设备同时在线，下个月重置流量额度\n - `34.99元/年`，每月有500GB的可用额度，允许最多15个设备同时在线，下个月重置流量额度\n - `68.99元/年`，每个月1000GB的可用额度，允许最多20个设备同时在线，下个月重置流量额度\n - 其余可以自行查看\n\n `赔钱机场`的订阅共有5种方案(按周期付费)，这里我仅显示自己正在使用的，个人认为十分优惠：\n - `34.99元/年`,每月500GB的可用额度，根据我观察和使用，这个订阅方案比`一元机场`的性价比更高，且流量使用额度也不用担心\n \n### 如何订阅？\n{% note success %}\n由于需要外部网络才能完成订阅服务的购买，你可以向我的邮箱`15202078626@163.com`发送你的订阅计划方案，\n扫描付款二维码，我将为你开通订阅（您只需要付款对应的订阅金额即可）\n{% endnote %}\n![](\"/img/yiyuan.png\")
![](\"/img/peiqian.png\")