Kosta 클라우드 네이티브 어플리케이션 개발(CNA) 교육
Kosta 클라우드 네이티브 애플리케이션 개발 (CNA) 교육 21일차
youngbee
2024. 12. 17. 08:30
JAVA Crawling and Json Parsing
package com.lec;
import java.io.IOException;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import com.myjdbc.DBManager;
import com.myjdbc.OracleDBManager;
public class chap16_Crawling2 {
public static void main(String[] args) {
chap16_Crawling2 o = new chap16_Crawling2();
o.insert(o.initDB());
o.select().forEach(i -> System.out.println(i));
o.delete();
}
public List<YtnClass> initDB() {
chap16_Crawling2 o = new chap16_Crawling2();
String url = "link";
List<YtnClass> list = new ArrayList<>();
try {
Document doc = Jsoup.connect(url).get();
Elements divs = doc
.select("div");
for (Element div : divs) {
String title = div.select("div.text_area > div.title > a").text();
String href = div.select("div.text_area > div.title > a").attr("href");
String image = div.select("div.photo > a > img").attr("data-src");
String regdate = div.select("div.text_area > div.info > div.date").text();
YtnClass y = new YtnClass(title, href, image, regdate);
list.add(y);
}
// list.stream().forEach(System.out::println);
} catch (IOException e) {
e.printStackTrace();
}
return list;
}
public int delete() {
DBManager dbm = OracleDBManager.getInstance();
Connection conn = dbm.connect();
PreparedStatement pstmt = null;
int rows = 0;
try {
String sql = "delete from ytntable";
pstmt = conn.prepareStatement(sql);
rows = pstmt.executeUpdate();
} catch (SQLException e) {
e.printStackTrace();
} finally {
dbm.close(conn, pstmt);
}
System.out.println("----------- 삭제 완료 -----------");
return rows;
}
public List<YtnClass> select() {
List<YtnClass> list = new ArrayList<YtnClass>();
DBManager dbm = OracleDBManager.getInstance();
Connection conn = dbm.connect();
PreparedStatement pstmt = null;
ResultSet rs = null;
try {
String sql = "select * from ytntable";
pstmt = conn.prepareStatement(sql);
rs = pstmt.executeQuery();
while(rs.next()) {
YtnClass y = new YtnClass();
y.setTitle(rs.getString("title"));
y.setHref(rs.getString("href"));
y.setImage(rs.getString("image"));
y.setRegdate(rs.getString("regdate"));
list.add(y);
}
} catch (SQLException e) {
e.printStackTrace();
} finally {
dbm.close(conn, pstmt, rs);
}
return list;
}
public void insert(List<YtnClass> list) {
String sql = "insert into ytntable(yseq, title, href, image, regdate) values(yseq_ytntable.nextval,?,?,?,?)";
DBManager dbm = OracleDBManager.getInstance();
try (Connection conn = dbm.connect();
PreparedStatement pstmt = conn.prepareStatement(sql)) {
for(YtnClass y : list) {
pstmt.setString(1, y.getTitle());
pstmt.setString(2, y.getHref());
pstmt.setString(3, y.getImage());
pstmt.setString(4, y.getRegdate());
pstmt.addBatch(); // 배치에 추가
}
pstmt.executeBatch(); // 한 번에 실행
System.out.println(" Insert 완료 ");
} catch (SQLException e) {
e.printStackTrace();
}
}
}
package com.lec;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
public class chap17_PublicPortal {
private static String SERVICE_KEY = "my key1";
private static String SERVICE_KEY2 = "my key2";
private static String apiUrl = "https://api.odcloud.kr/api/15004394/v1/uddi:422978ec-eaeb-433b-819d-eff597ed9bbe_201512281445?page=1&perPage=10&serviceKey="
+ SERVICE_KEY;
private static String apiUrl2 = "https://api.odcloud.kr/api/15004394/v1/uddi:422978ec-eaeb-433b-819d-eff597ed9bbe_201512281445?page=1&perPage=10";
public static void main(String[] args) {
// String key = "393b0243dc512c59009d10ba55cb1aa1";
chap17_PublicPortal cp = new chap17_PublicPortal();
String jsonData = cp.getMapJson1();
try {
ObjectMapper mapper = new ObjectMapper();
LinkedHashMap resMap;
resMap = mapper.readValue(jsonData, LinkedHashMap.class);
List<Map<String, String>> list = (List<Map<String, String>>) resMap.get("data");
ArrayList<DataVO> alist = new ArrayList<>();
for(Map<String, String> maps : list){
alist.add(
new DataVO(maps.get("20인이상단체 감면인원"),
maps.get("50인이상단체 감면인원"),
maps.get("N S K 감면인원"))
);
}
int ccnt = Integer.valueOf(String.valueOf(resMap.get("currentCount")));
int mcnt = Integer.valueOf(String.valueOf(resMap.get("matchCount")));
int page = Integer.valueOf(String.valueOf(resMap.get("page")));
int perPage = Integer.valueOf(String.valueOf(resMap.get("perPage")));
int total = Integer.valueOf(String.valueOf(resMap.get("totalCount")));
DataSetVO vo = new DataSetVO(alist, ccnt, mcnt, page, perPage, total);
System.out.println(vo);
} catch (JsonProcessingException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
public String getMapJson1() {
BufferedReader br = null;
StringBuilder sb = new StringBuilder();
try {
URL url = new URL(apiUrl);
br = new BufferedReader(new InputStreamReader(url.openStream()));
String line = "";
while ((line = br.readLine()) != null) {
sb.append(line);
}
br.close();
} catch (MalformedURLException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
// System.out.println(sb.toString());
return sb.toString();
}
public void getMapJson2() {
try {
URL url2 = new URL(apiUrl2);
HttpURLConnection conn = (HttpURLConnection) url2.openConnection();
conn.setRequestMethod("GET");
conn.setRequestProperty("Authorization", "Infuser " + SERVICE_KEY2);
System.out.println("Response code: " + conn.getResponseCode());
BufferedReader rd;
if (conn.getResponseCode() >= 200 && conn.getResponseCode() <= 300) {
rd = new BufferedReader(new InputStreamReader(conn.getInputStream()));
} else {
rd = new BufferedReader(new InputStreamReader(conn.getErrorStream()));
}
StringBuilder sb = new StringBuilder();
String line;
while ((line = rd.readLine()) != null) {
sb.append(line);
}
rd.close();
conn.disconnect();
// System.out.println(sb.toString());
} catch (IOException e) {
e.printStackTrace();
}
}
public void getMapJson3() {
try {
URL url = new URL(apiUrl);
BufferedReader br = new BufferedReader(new InputStreamReader(url.openStream()));
String line = "";
StringBuilder sb = new StringBuilder();
while ((line = br.readLine()) != null) {
sb.append(line);
}
br.close();
// System.out.println(sb.toString());
ObjectMapper mapper = new ObjectMapper();
LinkedHashMap resMap = mapper.readValue(sb.toString(), LinkedHashMap.class);
System.out.println(resMap.get("currentCount"));
List<Map<String, String>> list = (List<Map<String, String>>) resMap.get("data");
// list.forEach(i -> System.out.println("key = " + i.keySet()+ "\t" + "Value = "+i.values()));
ObjectMapper objectMapper = new ObjectMapper();
JsonNode rootNode = objectMapper.readTree("data");
String dataValue = rootNode.path("response").path("get").path("data").asText();
System.out.println(dataValue); // "desired value" 출력
} catch (MalformedURLException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
public void kakaoTest() {
Map<String, Object> map = new HashMap<>();
List<Long> list = new ArrayList<>();
list.add(11L);
list.add(22L);
list.add(33L);
map.put("elements", list);
map.put("elements", list);
map.put("before_url",
"https://kapi.kakao.com/v1/user/ids?limit=3&order=desc&from_id=1376016924426111111&app_key=12345674ae6e12379d5921f4417b399e7");
map.put("after_url",
"https://kapi.kakao.com/v1/user/ids?limit=3&order=asc&from_id=1376016924426333333&app_key=12345674ae6e12379d5921f4417b399e7");
}
}