Skip to content

J
jgjk
提交 73c604a2 authored 作者: 任建宇's avatar 任建宇
浏览文件
操作

shan

上级 29ae01b4
隐藏空白字符变更
内嵌 并排
正在显示 包含 375 行增加 和 52 行删除
app/resource/src/main/java/com/zrqx/resource/bg/mapper/shopCommodity/ShopCommodityMapper.java
浏览文件 @ 73c604a2
...@@ -43,7 +43,7 @@ public interface ShopCommodityMapper extends BaseMapper<ShopCommodity> { ...@@ -43,7 +43,7 @@ public interface ShopCommodityMapper extends BaseMapper<ShopCommodity> {
+ "#{id}" + "#{id}"
+ "</foreach>" + "</foreach>"
+ "</if>" + "</if>"
+ " ORDER BY (case when currentprice >=sellingprice then 2 else 1 end)asc, bookname desc" + " ORDER BY (case when currentprice >=sellingprice then 2 else 1 end)asc, bookname desc,a.id"
+ "</script>") + "</script>")
List<ShopCommodityVo> page(@Param("form")ShopCommodityForm form); List<ShopCommodityVo> page(@Param("form")ShopCommodityForm form);
......
app/resource/src/main/java/com/zrqx/resource/bg/service/shopCommodity/ShopCommoditySerivceImpl.java
浏览文件 @ 73c604a2
package com.zrqx.resource.bg.service.shopCommodity; package com.zrqx.resource.bg.service.shopCommodity;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.math.BigDecimal; import java.math.BigDecimal;
import java.net.URL; import java.net.URL;
import java.util.Date; import java.util.Date;
import java.util.List; import java.util.List;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.http.Consts;
import org.apache.http.HttpEntity;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
...@@ -15,6 +27,7 @@ import org.springframework.beans.factory.annotation.Autowired; ...@@ -15,6 +27,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSONObject;
import com.zrqx.core.enums.ResponseCodeEnum; import com.zrqx.core.enums.ResponseCodeEnum;
import com.zrqx.core.exception.BaseException; import com.zrqx.core.exception.BaseException;
...@@ -57,7 +70,7 @@ public class ShopCommoditySerivceImpl extends BaseServiceImpl<ShopCommodity, Str ...@@ -57,7 +70,7 @@ public class ShopCommoditySerivceImpl extends BaseServiceImpl<ShopCommodity, Str
@Override @Override
public PageInfo<ShopCommodityVo> page(ShopCommodityForm form, PageParam pageParam) { public PageInfo<ShopCommodityVo> page(ShopCommodityForm form, PageParam pageParam) {
startPage(pageParam); startPage(pageParam);
List<ShopCommodityVo> list = mapper.page(form); List<ShopCommodityVo> list = mapper.page(form);
return new PageInfo<ShopCommodityVo>(list); return new PageInfo<ShopCommodityVo>(list);
...@@ -76,7 +89,8 @@ public class ShopCommoditySerivceImpl extends BaseServiceImpl<ShopCommodity, Str ...@@ -76,7 +89,8 @@ public class ShopCommoditySerivceImpl extends BaseServiceImpl<ShopCommodity, Str
if (shop.getPlatform().equals("当当1")) { if (shop.getPlatform().equals("当当1")) {
List<Commodity> listCommdoity = commodityService.selectAll(); List<Commodity> listCommdoity = commodityService.selectAll();
for (Commodity commdoity : listCommdoity) { for (Commodity commdoity : listCommdoity) {
String url = "http://shop.dangdang.com/14937/list.html?key="+commdoity.getIsbn();
String url = shop.getUrlWebsite() + "/list.html?key=" + commdoity.getIsbn();
try { try {
Document doc = Jsoup.parse(new URL(url), 300000); Document doc = Jsoup.parse(new URL(url), 300000);
// 图书信息 // 图书信息
...@@ -95,28 +109,29 @@ public class ShopCommoditySerivceImpl extends BaseServiceImpl<ShopCommodity, Str ...@@ -95,28 +109,29 @@ public class ShopCommoditySerivceImpl extends BaseServiceImpl<ShopCommodity, Str
for (Element element : li) { for (Element element : li) {
// 获取标题 // 获取标题
String title = element.getElementsByTag("a").attr("title"); String title = element.getElementsByTag("a").attr("title");
if(title.contains(commdoity.getIsbn())) { if (title.contains(commdoity.getIsbn())) {
// 获取价格 // 获取价格
String price = element.getElementsByClass("preis").text(); String price = element.getElementsByClass("preis").text();
Example example = this.createExample(); Example example = this.createExample();
example.createCriteria().andEqualTo("commodityId", commdoity.getId()).andEqualTo("shopId", shop.getId()); example.createCriteria().andEqualTo("commodityId", commdoity.getId())
List<ShopCommodity> listShop = this.selectByExample(example); .andEqualTo("shopId", shop.getId());
ShopCommodity shopCommodity = new ShopCommodity(); List<ShopCommodity> listShop = this.selectByExample(example);
BigDecimal price1= new BigDecimal(price.substring(1, price.length())); ShopCommodity shopCommodity = new ShopCommodity();
shopCommodity.setCurrentprice(price1); BigDecimal price1 = new BigDecimal(price.substring(1, price.length()));
if(listShop.size() > 0) { shopCommodity.setCurrentprice(price1);
//修改售价 if (listShop.size() > 0) {
shopCommodity.setUpdateTime(new Date()); // 修改售价
this.UpdateByExampleSelective(shopCommodity, example); shopCommodity.setUpdateTime(new Date());
continue; this.UpdateByExampleSelective(shopCommodity, example);
} continue;
shopCommodity.setShopId(shop.getId()); }
shopCommodity.setCommodityId(commdoity.getId()); shopCommodity.setShopId(shop.getId());
shopCommodity.setUpdateTime(new Date()); shopCommodity.setCommodityId(commdoity.getId());
shopCommodity.setUploadTime(new Date()); shopCommodity.setUpdateTime(new Date());
this.insert(shopCommodity); shopCommodity.setUploadTime(new Date());
} this.insert(shopCommodity);
}
} }
} catch (Exception e) { } catch (Exception e) {
...@@ -125,48 +140,356 @@ public class ShopCommoditySerivceImpl extends BaseServiceImpl<ShopCommodity, Str ...@@ -125,48 +140,356 @@ public class ShopCommoditySerivceImpl extends BaseServiceImpl<ShopCommodity, Str
} }
} }
if (shop.getPlatform().equals("蔚蓝")) { if (shop.getPlatform().equals("蔚蓝1")) {
List<Commodity> listCommdoity = commodityService.selectAll(); List<Commodity> listCommdoity = commodityService.selectAll();
for (Commodity commdoity : listCommdoity) { for (Commodity commdoity : listCommdoity) {
String url = "https://www.welan.com/search/ajax?keywords="; String url = "https://www.welan.com/search/ajax?keywords=";
String result=HttpsUtils.sendGet(url,commdoity.getIsbn()); String result = HttpsUtils.sendGet(url, commdoity.getIsbn());
JSONObject object = (JSONObject)JSON.parse(result); JSONObject object = (JSONObject) JSON.parse(result);
//唯一标识用户的 openId try {
try {
Thread.sleep(2000); Thread.sleep(2000);
} catch (InterruptedException e1) { } catch (InterruptedException e1) {
// TODO Auto-generated catch block // TODO Auto-generated catch block
e1.printStackTrace(); e1.printStackTrace();
} }
if(Integer.parseInt(object.get("dataCount").toString()) > 0) { if (Integer.parseInt(object.get("dataCount").toString()) > 0) {
System.out.println(); System.out.println();
if(commdoity.getIsbn().equals(object.getJSONArray("books").getJSONObject(0).get("isbn"))) { if (commdoity.getIsbn().equals(object.getJSONArray("books").getJSONObject(0).get("isbn"))) {
Example example = this.createExample();
example.createCriteria().andEqualTo("commodityId", commdoity.getId()).andEqualTo("shopId",
shop.getId());
List<ShopCommodity> listShop = this.selectByExample(example);
ShopCommodity shopCommodity = new ShopCommodity();
BigDecimal price1 = new BigDecimal(
object.getJSONArray("books").getJSONObject(0).get("wlPrice").toString());
shopCommodity.setCurrentprice(price1);
if (listShop.size() > 0) {
// 修改售价
shopCommodity.setUpdateTime(new Date());
this.UpdateByExampleSelective(shopCommodity, example);
continue;
}
shopCommodity.setShopId(shop.getId());
shopCommodity.setCommodityId(commdoity.getId());
shopCommodity.setUpdateTime(new Date());
shopCommodity.setUploadTime(new Date());
this.insert(shopCommodity);
}
}
}
}
if (shop.getPlatform().equals("文轩网1")) {
List<Commodity> listCommdoity = commodityService.selectAll();
for (Commodity commdoity : listCommdoity) {
String url = "https://search.winxuan.com/search?keyword=" + commdoity.getIsbn();
try {
Document doc = Jsoup.parse(new URL(url), 300000);
// 图书信息
Elements elements = doc.getElementsByTag("ul");
// System.out.println(elements);
Element tempElement = null;
for (Element element : elements) {
if (element.className().equals("group-ul")) {
tempElement = element;
// System.out.println(element.className());
break;
}
}
System.out.println(tempElement);
Elements li = tempElement.getElementsByTag("li");
for (Element element : li) {
// 获取标题
String title = element.getElementsByTag("a").attr("title");
// 获取价格
String price = element.getElementsByClass("price-n").text();
Example example = this.createExample();
example.createCriteria().andEqualTo("commodityId", commdoity.getId()).andEqualTo("shopId",
shop.getId());
List<ShopCommodity> listShop = this.selectByExample(example);
ShopCommodity shopCommodity = new ShopCommodity();
BigDecimal price1 = new BigDecimal(price.substring(1, price.length()));
shopCommodity.setCurrentprice(price1);
if (listShop.size() > 0) {
// 修改售价
shopCommodity.setUpdateTime(new Date());
this.UpdateByExampleSelective(shopCommodity, example);
continue;
}
shopCommodity.setShopId(shop.getId());
shopCommodity.setCommodityId(commdoity.getId());
shopCommodity.setUpdateTime(new Date());
shopCommodity.setUploadTime(new Date());
this.insert(shopCommodity);
}
} catch (Exception e) {
// TODO: handle exception
}
}
}
if (shop.getPlatform().equals("孔夫子1")) {
List<Commodity> listCommdoity = commodityService.selectAll();
for (Commodity commdoity : listCommdoity) {
String url = shop.getUrlWebsite() + "/search/?key=" + commdoity.getIsbn();
try {
Document doc = Jsoup.parse(new URL(url), 300000);
// 图书信息
Elements elements = doc.select("span[class=bold]");
if (elements != null) {
for (Element element : elements) {
// 获取价格
String price = element.getElementsByClass("bold").text();
Example example = this.createExample(); Example example = this.createExample();
example.createCriteria().andEqualTo("commodityId", commdoity.getId()).andEqualTo("shopId", shop.getId()); example.createCriteria().andEqualTo("commodityId", commdoity.getId())
List<ShopCommodity> listShop = this.selectByExample(example); .andEqualTo("shopId", shop.getId());
ShopCommodity shopCommodity = new ShopCommodity(); List<ShopCommodity> listShop = this.selectByExample(example);
BigDecimal price1= new BigDecimal(object.getJSONArray("books").getJSONObject(0).get("wlPrice").toString()); ShopCommodity shopCommodity = new ShopCommodity();
shopCommodity.setCurrentprice(price1); BigDecimal price1 = new BigDecimal(price.substring(1, price.length()));
if(listShop.size() > 0) { shopCommodity.setCurrentprice(price1);
//修改售价 if (listShop.size() > 0) {
shopCommodity.setUpdateTime(new Date()); // 修改售价
this.UpdateByExampleSelective(shopCommodity, example); shopCommodity.setUpdateTime(new Date());
continue; this.UpdateByExampleSelective(shopCommodity, example);
} continue;
shopCommodity.setShopId(shop.getId()); }
shopCommodity.setCommodityId(commdoity.getId()); shopCommodity.setShopId(shop.getId());
shopCommodity.setUpdateTime(new Date()); shopCommodity.setCommodityId(commdoity.getId());
shopCommodity.setUploadTime(new Date()); shopCommodity.setUpdateTime(new Date());
this.insert(shopCommodity); shopCommodity.setUploadTime(new Date());
} this.insert(shopCommodity);
} break;
}
}
} catch (Exception e) {
// TODO: handle exception
}
}
}
//爬取苏宁易购商品销售价格
if (shop.getPlatform().equals("苏宁易购1")) {
List<Commodity> listCommdoity = commodityService.selectAll();
for (Commodity commdoity : listCommdoity) {
try {
Document doc = Jsoup.parse(new URL(shop.getUrlWebsite()), 300000);
// 图书信息
Element element = doc.getElementById("CSupplierID");
// 获取商店信息ID
String shopId = "00"+element.val();
// 拼接接口路径
String url = "https://csearch.suning.com/emall/cshop/queryByKeyword.do?vendor_Id=" + shopId
+ "&keyword=" + commdoity.getIsbn() + "&start=0&rows=48&sortField=&cf=channel:PC,price:&pcode=11749307328&cityId=010&custno=&callback=jsonpQueryByKeyword";
String result = HttpsUtils.sendGet(url, commdoity.getIsbn());
System.out.println(result);
result = result.substring(34,result.length()-2);
JSONObject object = (JSONObject) JSON.parse(result);
System.out.println(object.get("goodsSize").toString());
//睡眠2秒
Thread.sleep(2000);
if (Integer.parseInt(object.get("goodsSize").toString()) > 0) {
JSONArray jsonarry = object.getJSONArray("goods");
for (int i = 0; i < jsonarry.size(); i++) {
Example example = this.createExample();
example.createCriteria().andEqualTo("commodityId", commdoity.getId())
.andEqualTo("shopId", shop.getId());
List<ShopCommodity> listShop = this.selectByExample(example);
ShopCommodity shopCommodity = new ShopCommodity();
BigDecimal price1 = new BigDecimal(
jsonarry.getJSONObject(i).get("price").toString());
shopCommodity.setCurrentprice(price1.setScale(2, BigDecimal.ROUND_HALF_UP));
if (listShop.size() > 0) {
// 修改售价
shopCommodity.setUpdateTime(new Date());
this.UpdateByExampleSelective(shopCommodity, example);
continue;
}
shopCommodity.setShopId(shop.getId());
shopCommodity.setCommodityId(commdoity.getId());
shopCommodity.setUpdateTime(new Date());
shopCommodity.setUploadTime(new Date());
this.insert(shopCommodity);
}
}
} catch (Exception e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
}
}
if (shop.getPlatform().equals("网易考拉1")) {
List<Commodity> listCommdoity = commodityService.selectAll();
for (Commodity commdoity : listCommdoity) {
//睡眠2秒
String url = shop.getUrlWebsite();
String shopid = null;
if(url.contains("pc")) {
shopid = url.substring(url.indexOf("pc/")+3, url.indexOf("pc/")+12);
}else {
shopid = url.substring(url.indexOf("kaola.com/")+10, url.indexOf("kaola.com/")+19);
}
url = "https://mall.kaola.com/search.html?shopId=" + shopid +"&key="+commdoity.getIsbn();
try {
Thread.sleep(2000);
Document doc = Jsoup.parse(new URL(url), 300000);
// 图书信息
Elements elements = doc.getElementsByTag("ul");
// System.out.println(elements);
Element tempElement = null;
for (Element element : elements) {
if (element.className().equals("clearfix")) {
tempElement = element;
// System.out.println(element.className());
break;
}
}
System.out.println(tempElement);
Elements li = tempElement.getElementsByTag("li");
for (Element element : li) {
// 获取价格
String price = element.getElementsByClass("cur").text();
Example example = this.createExample();
example.createCriteria().andEqualTo("commodityId", commdoity.getId()).andEqualTo("shopId",
shop.getId());
List<ShopCommodity> listShop = this.selectByExample(example);
ShopCommodity shopCommodity = new ShopCommodity();
BigDecimal price1 = new BigDecimal(price.substring(1, price.length()));
shopCommodity.setCurrentprice(price1);
if (listShop.size() > 0) {
// 修改售价
shopCommodity.setUpdateTime(new Date());
this.UpdateByExampleSelective(shopCommodity, example);
continue;
}
shopCommodity.setShopId(shop.getId());
shopCommodity.setCommodityId(commdoity.getId());
shopCommodity.setUpdateTime(new Date());
shopCommodity.setUploadTime(new Date());
this.insert(shopCommodity);
}
} catch (Exception e) {
// TODO: handle exception
}
} }
} }
if (shop.getPlatform().equals("京东")) {
List<Commodity> listCommdoity = commodityService.selectAll();
for (Commodity commdoity : listCommdoity) {
// String url = shop.getUrlWebsite();
int j=10;
for (int i = 0; i < j; i++) {
String input = commdoity.getIsbn();
// 需要爬取商品信息的网站地址
String url = "https://search.jd.com/Search?keyword=" + input + "&enc=utf-8";
// 动态模拟请求数据
CloseableHttpClient httpclient = HttpClients.createDefault();
HttpGet httpGet = new HttpGet(url);
// 模拟浏览器浏览（user-agent的值可以通过浏览器浏览，查看发出请求的头文件获取）
httpGet.setHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36");
CloseableHttpResponse response = null;
try {
Thread.sleep(5000);
response = httpclient.execute(httpGet);
// 获取响应状态码
int statusCode = response.getStatusLine().getStatusCode();
HttpEntity entity = response.getEntity();
// 如果状态响应码为200，则获取html实体内容或者json文件
if (statusCode == 200) {
String html = EntityUtils.toString(entity, Consts.UTF_8);
// 提取HTML得到商品信息结果
Document doc = Jsoup.parse(html);
File f=new File("D:\\out.txt");
f.createNewFile();
FileOutputStream fileOutputStream = new FileOutputStream(f);
PrintStream printStream = new PrintStream(fileOutputStream);
System.setOut(printStream);
System.out.println("默认输出到控制台的这一句，输出到了文件 out.txt");
System.out.println(doc);
// 通过浏览器查看商品页面的源代码，找到信息所在的div标签，再对其进行一步一步地解析,这都需要对html代码进行分析了
Elements ulList = doc.select("#J_goodsList");
Elements liList = ulList.select(".gl-item");
Elements fptex = doc.select(".fp-text");
System.out.println(fptex.text());
if(StringUtils.isNotBlank(fptex.text())) {
String text = fptex.text().substring(2, 3);
j = Integer.parseInt(text);
System.out.println("j进来玩"+j);
}
// 循环liList的数据（具体获取的数据值还得看doc的页面源代码来获取，可能稍有变动）
for (Element item : liList) {
// 商品价格
String price = item.select(".p-price").select("i").text();
System.out.println("商品价格：" + price);
// 店铺url
String shopUrl = item.select(".J_im_icon").select("a").attr("href");
//截取店铺、
String shopUrlLocahost = shop.getUrlWebsite().substring(shop.getUrlWebsite().indexOf("index"), shop.getUrlWebsite().lastIndexOf("."));
System.out.println("------------------------------------");
//列表店铺存在将修改或增加今日爬取数据
if(shopUrl.contains(shopUrlLocahost)) {
Example example = this.createExample();
example.createCriteria().andEqualTo("commodityId", commdoity.getId()).andEqualTo("shopId",
shop.getId());
List<ShopCommodity> listShop = this.selectByExample(example);
ShopCommodity shopCommodity = new ShopCommodity();
BigDecimal price1 = new BigDecimal(price);
shopCommodity.setCurrentprice(price1);
if (listShop.size() > 0) {
// 修改售价
shopCommodity.setUpdateTime(new Date());
this.UpdateByExampleSelective(shopCommodity, example);
continue;
}
shopCommodity.setShopId(shop.getId());
shopCommodity.setCommodityId(commdoity.getId());
shopCommodity.setUpdateTime(new Date());
shopCommodity.setUploadTime(new Date());
this.insert(shopCommodity);
}
}
// 消耗掉实体
EntityUtils.consume(response.getEntity());
} else {
// 消耗掉实体
EntityUtils.consume(response.getEntity());
}
}catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
} finally {
try {
response.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
}
}
} }
return false; return false;
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论