.libPaths("d://R_package/R.4.2.1/")
# 加载必要的库
library(httr)
library(rvest)
library(stringr)
library(dplyr)
library(ggplot2)
library(ggprism)
library(purrr)
library(tidyr)
library(forcats)
library(plyr)
# 函数来获取页面内容
get_page_content <- function(url) {
Sys.sleep(5) # 添加延迟以避免被封IP
response <- GET(url, user_agent("Mozilla/5.0"))
content(response, "text")
}
# 函数来解析页面内容
parse_page <- function(html_content) {
page <- read_html(html_content)
title <- page %>% html_nodes('li.house-cell.realverify div.des h2 a') %>% html_text(trim = TRUE)
url <- page %>% html_nodes('li.house-cell.realverify div.des h2 a') %>% html_attr("href")
size <- page %>% html_nodes('li.house-cell.realverify div.des p.room') %>% html_text(trim = TRUE)
info <- page %>% html_nodes('li.house-cell.realverify div.des p.infor') %>% html_text(trim = TRUE)
price <- page %>% html_nodes('li.house-cell.realverify div.money') %>% html_text(trim = TRUE)
author <- page %>% html_nodes('li.house-cell.realverify div.jjr') %>% html_text(trim = TRUE)
data.frame(title = title, url = url, size = size, info = info, price = price, author = author)
}
# 获取所有页面的数据
zhengzu_info <- map_df(1:70, function(page_num) {
url <- paste0("https://nj.58.com/zufang/sub/l47/pn", page_num, "/")
html_content <- get_page_content(url)
df <- parse_page(html_content)
print(paste("完成第", page_num, "页信息提取，共获得", nrow(df), "条信息", sep = ""))
df
})
# 保存原始数据
saveRDS(zhengzu_info, "zhengzu.rds")
zhengzu_info_raw<-readRDS("zhengzu.rds")
# 数据清洗
yihaoxianzhan <- c("迈皋桥站", "红山动物园站", "南京站站", "新模范马路站",
"玄武门站", "鼓楼站", "珠江路站", "新街口站",
"张府园站", "三山街站", "中华门站", "安德门站",
"天隆寺站", "软件大道站", "花神庙站", "南京南站站",
"双龙大道站", "河定桥站", "胜太路站", "百家湖站",
"小龙湾站", "竹山路站", "天印大道站", "龙眠大道站",
"南医大站", "南京交院站", "中国药科大学站")
zhengzu_info <- zhengzu_info_raw %>%
mutate(
title = str_remove_all(title, " "),
num =str_split(size, " {1,}", simplify = TRUE)%>%.[,1],
area = str_split(size, " {1,}", simplify = TRUE)%>%.[,2],
info = str_remove_all(info, " {1,}"),
dir1 = str_split(info, "\\n{1,}", simplify = TRUE)[,1],
dir2 = str_split(info, "\\n{1,}", simplify = TRUE)[,3],
dir_tmp = str_split(info, "\\n{1,}", simplify = TRUE)[,4],
dir3 = str_remove(str_extract(dir_tmp, "距1号线-[\\p{Han}]{1,}"), "距1号线-"),
dir4 = as.numeric(str_remove(str_extract(dir_tmp, "[0-9]{1,}m"), "m")),
author = str_remove_all(author, " {1,}"),
author1 = str_split(author, "\\n{1,}", simplify = TRUE)[,1],
author2 = str_split(author, "\\n{1,}", simplify = TRUE)[,2],
author3 = str_split(author, "\\n{1,}", simplify = TRUE)[,3]
) %>%
select(title, num, area, dir1, dir2, dir3, dir4, price, author1, author2, author3, url) %>%
mutate(
area = as.numeric(str_extract(area, "[0-9\\.]{1,}")),
price = as.numeric(str_remove(price, "元/月"))
)
yihaoxianzhan <- intersect(yihaoxianzhan, unique(paste0(zhengzu_info$dir3,"站")))
zhengzu_info$dir3 <- factor(paste0(zhengzu_info$dir3,"站"), levels = yihaoxianzhan)
# 保存清洗后的数据
saveRDS(zhengzu_info, "zhengzu_info_cleandata.rds")
# 去除重复项
zhengzu_info <- zhengzu_info[!duplicated(zhengzu_info$title),]
#统计
#1、计算均房价
zhengzu_summary<-zhengzu_info%>%
mutate(ave_price=price/area)%>%
filter(!is.na(dir3))%>%
mutate(dir3=factor(dir3,levels = yihaoxianzhan))
zhengzu_num<-ddply(zhengzu_summary,.(dir3),nrow)
ggplot(zhengzu_summary%>%filter(!is.na(dir3)),aes(x=dir3,y=ave_price))+
geom_boxplot()+
geom_text(data = zhengzu_num,aes(x=dir3,y=120,label=paste0("n=",V1)))+
#geom_jitter()+
coord_flip()+
scale_y_continuous(limits = c(15,150),breaks = seq(0,120,10))+
labs(x="",y="租房均价/m^2/月")
#根据均房价高低排序
zhengzu_summary_mean=zhengzu_summary%>%
ddply(.,.(dir3),function(x){mean(x$ave_price)})%>%
arrange(V1)
zhengzu_summary%>%
mutate(dir3=factor(dir3,levels = zhengzu_summary_mean$dir3))%>%
ggplot(aes(x=dir3,y=ave_price))+
geom_boxplot()+
geom_text(data = zhengzu_num,aes(x=dir3,y=120,label=paste0("n=",V1)))+
#geom_jitter()+
coord_flip()+
scale_y_continuous(limits = c(15,150),breaks = seq(0,120,10))+
labs(x="",y="租房均价/m^2/月")
#距离地铁站的距离与均价的关系
zhengzu_summary%>%
filter(!is.na(dir4))%>%
mutate(dir5=ifelse(dir4<500,"500米以内",ifelse(dir4>=500&dir4<1000,"500-1000米",ifelse(dir4>=1000&dir4<1500,"1000-1500米","大于1500米"))))%>%
mutate(dir5=factor(dir5,levels=c("500米以内","500-1000米","1000-1500米","大于1500米")))%>%
ggplot(aes(x=dir5,y=ave_price,color=dir3))+
geom_boxplot()+
scale_y_continuous(limits = c(15,150))+
theme(legend.position = "bottom")+
labs(x="到1号线站点的距离",y="租房均价/m^2/月")
zhengzu_summary%>%
ggplot(aes(x=area,y=price,color=dir3))+
geom_point()+
scale_x_continuous(limits = c(0,200))+
scale_y_continuous(limits = c(0,10000))
zhengzu_summary_classmean<-zhengzu_summary%>%
filter(!is.na(dir4))%>%
filter(!is.na(dir3))%>%
mutate(dir5=ifelse(dir4<500,"A",ifelse(dir4>=500&dir4<1000,"B",ifelse(dir4>=1000&dir4<1500,"C","D"))))%>%
ddply(.,.(dir5,dir3),function(x){
nrow(x)
})%>%
mutate(y=sapply(dir5,function(y){switch(y,"A"=25,"B"=20,"C"=15,"D"=10)}))
zhengzu_summary%>%
filter(!is.na(dir4))%>%
filter(!is.na(dir3))%>%
mutate(dir5=ifelse(dir4<500,"A",ifelse(dir4>=500&dir4<1000,"B",ifelse(dir4>=1000&dir4<1500,"C","D"))))%>%
ddply(.,.(dir3,dir5),function(x){mean(x$ave_price)})%>%
ggplot(aes(x=dir3,y=V1,color=dir5))+
geom_line(aes(color=dir5,group=dir5),size=2)+
geom_point(aes(color=dir5),size=5)+
geom_text(data = zhengzu_summary_classmean,aes(x=dir3,y=y,label=V1,color=dir5))+
#geom_text_repel(aes(label=round(V1,2)),max.overlaps = 100)+
scale_y_continuous(limits = c(5,90),breaks = seq(0,75,5))+
scale_color_discrete(label=c("0-500m","500-1000m","1000-1500m","1500m以上"),name="到地铁站距离")+
theme_prism()+
theme(axis.text.x = element_text(angle = 45,hjust = 1,vjust = 1),
legend.title = element_text())+
#facet_wrap(~factor(dir5),scales = "free_y")+
labs(y="租房均价/m^2",x="")
网友评论