|
|
|
|
|
using Npgsql;
|
|
|
|
|
|
using System;
|
|
|
|
|
|
using System.Collections.Generic;
|
|
|
|
|
|
using System.Linq;
|
|
|
|
|
|
using System.Text;
|
|
|
|
|
|
using System.Threading.Tasks;
|
|
|
|
|
|
using PhilExampleCrawler.DataBase.Models;
|
|
|
|
|
|
using System.Data;
|
|
|
|
|
|
|
|
|
|
|
|
namespace PhilExampleCrawler.DataBase
|
|
|
|
|
|
{
|
|
|
|
|
|
public class CrawlSessionAccess
|
|
|
|
|
|
{
|
|
|
|
|
|
private readonly string _dbconn;
|
|
|
|
|
|
|
|
|
|
|
|
public CrawlSessionAccess(DBSettings dbSettings)
|
|
|
|
|
|
{
|
|
|
|
|
|
_dbconn = dbSettings.ToString();
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
public async Task<DB_CrawlSession?> AddCrawlSessionAsync(DB_CrawlSession newCS)
|
|
|
|
|
|
{
|
|
|
|
|
|
DB_CrawlSession? cs = null;
|
|
|
|
|
|
try
|
|
|
|
|
|
{
|
|
|
|
|
|
using var conn = new NpgsqlConnection(_dbconn);
|
|
|
|
|
|
conn.Open();
|
|
|
|
|
|
|
|
|
|
|
|
var cmd = conn.CreateCommand();
|
|
|
|
|
|
cmd.CommandText = SQL_InsertCrawlSession(newCS);
|
|
|
|
|
|
if (await cmd.ExecuteScalarAsync() is int csID)
|
|
|
|
|
|
cs = new DB_CrawlSession(id: csID,
|
|
|
|
|
|
user_id: newCS.UserID,
|
|
|
|
|
|
keywords: newCS.Keywords,
|
|
|
|
|
|
location_text: newCS.LocationText,
|
|
|
|
|
|
category_id: newCS.CategoryID,
|
|
|
|
|
|
radius_km: newCS.RadiusKM,
|
|
|
|
|
|
minPrice: newCS.MinPrice,
|
|
|
|
|
|
maxPrice: newCS.MaxPrice,
|
|
|
|
|
|
isPrivate: newCS.IsPrivate,
|
|
|
|
|
|
isCommerial: newCS.IsCommercial);
|
|
|
|
|
|
|
|
|
|
|
|
conn.Close();
|
|
|
|
|
|
}
|
|
|
|
|
|
catch (Exception ex)
|
|
|
|
|
|
{
|
|
|
|
|
|
Console.WriteLine(ex);
|
|
|
|
|
|
//TODO: log
|
|
|
|
|
|
}
|
|
|
|
|
|
return cs;
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
public async Task<bool> UpdateCrawlSessionAsync(DB_CrawlSession cs)
|
|
|
|
|
|
{
|
|
|
|
|
|
try
|
|
|
|
|
|
{
|
|
|
|
|
|
using var conn = new NpgsqlConnection(_dbconn);
|
|
|
|
|
|
conn.Open();
|
|
|
|
|
|
|
|
|
|
|
|
var cmd = conn.CreateCommand();
|
|
|
|
|
|
cmd.CommandText = SQL_UpdateCrawlSession(cs);
|
|
|
|
|
|
if (await cmd.ExecuteNonQueryAsync() > 0)
|
|
|
|
|
|
return true;
|
|
|
|
|
|
|
|
|
|
|
|
conn.Close();
|
|
|
|
|
|
}
|
|
|
|
|
|
catch (Exception ex)
|
|
|
|
|
|
{
|
|
|
|
|
|
Console.WriteLine(ex);
|
|
|
|
|
|
//TODO: log
|
|
|
|
|
|
}
|
|
|
|
|
|
return false;
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
public async Task<List<DB_CrawlSession>> GetCrawlSessionsAsync(int userID)
|
|
|
|
|
|
{
|
|
|
|
|
|
List<DB_CrawlSession> l = new();
|
|
|
|
|
|
try
|
|
|
|
|
|
{
|
|
|
|
|
|
using var conn = new NpgsqlConnection(_dbconn);
|
|
|
|
|
|
conn.Open();
|
|
|
|
|
|
|
|
|
|
|
|
var cmd = conn.CreateCommand();
|
|
|
|
|
|
cmd.CommandText = SQL_SeletCrawlSession(userID);
|
|
|
|
|
|
using (var dr = await cmd.ExecuteReaderAsync())
|
|
|
|
|
|
while (await dr.ReadAsync())
|
|
|
|
|
|
l.Add(new DB_CrawlSession(id: dr.GetInt32("id"),
|
|
|
|
|
|
user_id: userID,
|
|
|
|
|
|
keywords: dr.GetString("keywords"),
|
|
|
|
|
|
location_text: dr.GetString("location_text"),
|
|
|
|
|
|
category_id: dr.GetInt32("category_id"),
|
|
|
|
|
|
radius_km: dr.GetInt32("radius_km"),
|
|
|
|
|
|
minPrice: dr.GetInt32("min_price"),
|
|
|
|
|
|
maxPrice: dr.GetInt32("max_price"),
|
|
|
|
|
|
isPrivate: dr.GetBoolean("is_private"),
|
|
|
|
|
|
isCommerial: dr.GetBoolean("is_commercial")));
|
|
|
|
|
|
|
|
|
|
|
|
conn.Close();
|
|
|
|
|
|
}
|
|
|
|
|
|
catch (Exception ex)
|
|
|
|
|
|
{
|
|
|
|
|
|
Console.WriteLine(ex);
|
|
|
|
|
|
//TODO: log
|
|
|
|
|
|
}
|
|
|
|
|
|
return l;
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
public async Task<List<DB_CrawlSession>> GetAllCrawlSessionsAsync()
|
|
|
|
|
|
{
|
|
|
|
|
|
List<DB_CrawlSession> l = new();
|
|
|
|
|
|
try
|
|
|
|
|
|
{
|
|
|
|
|
|
using var conn = new NpgsqlConnection(_dbconn);
|
|
|
|
|
|
conn.Open();
|
|
|
|
|
|
|
|
|
|
|
|
var cmd = conn.CreateCommand();
|
|
|
|
|
|
cmd.CommandText = SQL_SeletAllCrawlSessions();
|
|
|
|
|
|
using (var dr = await cmd.ExecuteReaderAsync())
|
|
|
|
|
|
while (await dr.ReadAsync())
|
|
|
|
|
|
l.Add(new DB_CrawlSession(id: dr.GetInt32("id"),
|
|
|
|
|
|
user_id: dr.GetInt32("user_id"),
|
|
|
|
|
|
keywords: dr.GetString("keywords"),
|
|
|
|
|
|
location_text: dr.GetString("location_text"),
|
|
|
|
|
|
category_id: dr.GetInt32("category_id"),
|
|
|
|
|
|
radius_km: dr.GetInt32("radius_km"),
|
|
|
|
|
|
minPrice: dr.GetInt32("min_price"),
|
|
|
|
|
|
maxPrice: dr.GetInt32("max_price"),
|
|
|
|
|
|
isPrivate: dr.GetBoolean("is_private"),
|
|
|
|
|
|
isCommerial: dr.GetBoolean("is_commercial")));
|
|
|
|
|
|
|
|
|
|
|
|
conn.Close();
|
|
|
|
|
|
}
|
|
|
|
|
|
catch (Exception ex)
|
|
|
|
|
|
{
|
|
|
|
|
|
Console.WriteLine(ex);
|
|
|
|
|
|
//TODO: log
|
|
|
|
|
|
}
|
|
|
|
|
|
return l;
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
public async Task<bool> RemoveCrawlSessionAsync(int crawlSessionID)
|
|
|
|
|
|
{
|
|
|
|
|
|
bool deleted = false;
|
|
|
|
|
|
try
|
|
|
|
|
|
{
|
|
|
|
|
|
using var conn = new NpgsqlConnection(_dbconn);
|
|
|
|
|
|
conn.Open();
|
|
|
|
|
|
|
|
|
|
|
|
var cmd = conn.CreateCommand();
|
|
|
|
|
|
cmd.CommandText = SQL_DeleteCrawlSession(crawlSessionID);
|
|
|
|
|
|
await cmd.ExecuteNonQueryAsync();
|
|
|
|
|
|
deleted = true;
|
|
|
|
|
|
|
|
|
|
|
|
conn.Close();
|
|
|
|
|
|
}
|
|
|
|
|
|
catch (Exception ex)
|
|
|
|
|
|
{
|
|
|
|
|
|
Console.WriteLine(ex);
|
|
|
|
|
|
//TODO: log
|
|
|
|
|
|
}
|
|
|
|
|
|
return deleted;
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
public async Task<List<DB_Insertion>> AddInsertionsAsync(IEnumerable<DB_Insertion> insertions)
|
|
|
|
|
|
{
|
|
|
|
|
|
List<DB_Insertion> l = new();
|
|
|
|
|
|
|
|
|
|
|
|
if (!insertions.Any())
|
|
|
|
|
|
return l;
|
|
|
|
|
|
|
|
|
|
|
|
try
|
|
|
|
|
|
{
|
|
|
|
|
|
using var conn = new NpgsqlConnection(_dbconn);
|
|
|
|
|
|
conn.Open();
|
|
|
|
|
|
|
|
|
|
|
|
var cmd = conn.CreateCommand();
|
|
|
|
|
|
foreach(var i in insertions)
|
|
|
|
|
|
{
|
|
|
|
|
|
cmd.CommandText = SQL_InsertInsertion(i.Href, i.CrawlSessionID, i.Name, i.PostCode, i.LocationStr, i.Price, i.Is_VB,
|
|
|
|
|
|
i.Date, i.IsTopAd, i.IsHighlight, i.IsRequest);
|
|
|
|
|
|
if (await cmd.ExecuteNonQueryAsync() > 0)
|
|
|
|
|
|
l.Add(new DB_Insertion(i));
|
|
|
|
|
|
}
|
|
|
|
|
|
conn.Close();
|
|
|
|
|
|
}
|
|
|
|
|
|
catch (Exception ex)
|
|
|
|
|
|
{
|
|
|
|
|
|
Console.WriteLine(ex);
|
|
|
|
|
|
//TODO: log
|
|
|
|
|
|
}
|
|
|
|
|
|
return l;
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
public async Task<List<DB_Insertion>> GetInsertions(int userID, int? limitPerCrawlSession = 5)
|
|
|
|
|
|
{
|
|
|
|
|
|
List<DB_Insertion> ins = new();
|
|
|
|
|
|
try
|
|
|
|
|
|
{
|
|
|
|
|
|
using var conn = new NpgsqlConnection(_dbconn);
|
|
|
|
|
|
conn.Open();
|
|
|
|
|
|
|
|
|
|
|
|
var cmd = conn.CreateCommand();
|
|
|
|
|
|
cmd.CommandText = SQL_SeletAllCrawlSessionIDs(userID);
|
|
|
|
|
|
|
|
|
|
|
|
//crawl session ids
|
|
|
|
|
|
List<int> csIDs = new();
|
|
|
|
|
|
using (var dr = await cmd.ExecuteReaderAsync())
|
|
|
|
|
|
while (await dr.ReadAsync())
|
|
|
|
|
|
csIDs.Add(dr.GetInt32("id"));
|
|
|
|
|
|
|
|
|
|
|
|
//insertions
|
|
|
|
|
|
foreach (int csID in csIDs)
|
|
|
|
|
|
{
|
|
|
|
|
|
cmd.CommandText = SQL_SelectInsertions(csID, limitPerCrawlSession);
|
|
|
|
|
|
using var dr = await cmd.ExecuteReaderAsync();
|
|
|
|
|
|
while (await dr.ReadAsync())
|
|
|
|
|
|
ins.Add(new DB_Insertion(href: dr.GetString("href"),
|
|
|
|
|
|
crawlSessionID: csID,
|
|
|
|
|
|
name: dr.GetString("name"),
|
|
|
|
|
|
postCode: dr.ReadNullableInt("post_code"),
|
|
|
|
|
|
locationStr: dr.ReadNullableString("location_text"),
|
|
|
|
|
|
price: dr.GetDecimal("price"),
|
|
|
|
|
|
is_vb: dr.GetBoolean("is_vb"),
|
|
|
|
|
|
date: dr.ReadNullableDateTime("date"),
|
|
|
|
|
|
isTopAd: dr.GetBoolean("is_topad"),
|
|
|
|
|
|
isHighlight: dr.GetBoolean("is_highlight"),
|
|
|
|
|
|
isRequest: dr.GetBoolean("is_request")));
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
conn.Close();
|
|
|
|
|
|
}
|
|
|
|
|
|
catch (Exception ex)
|
|
|
|
|
|
{
|
|
|
|
|
|
Console.WriteLine(ex);
|
|
|
|
|
|
//TODO: log
|
|
|
|
|
|
}
|
|
|
|
|
|
return ins;
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
#region SQLQueries
|
|
|
|
|
|
private static string SQL_InsertCrawlSession (DB_CrawlSession cs)
|
|
|
|
|
|
=> $"INSERT INTO " +
|
|
|
|
|
|
"crawl_session" +
|
|
|
|
|
|
"(user_id, keywords, location_text, category_id, radius_km, min_price, max_price, is_private, is_commercial)" +
|
|
|
|
|
|
" VALUES " +
|
|
|
|
|
|
"(" +
|
|
|
|
|
|
$"{cs.UserID}," +
|
|
|
|
|
|
$"'{cs.Keywords}'," +
|
|
|
|
|
|
$"'{cs.LocationText}'," +
|
|
|
|
|
|
$"{cs.CategoryID}," +
|
|
|
|
|
|
$"{cs.RadiusKM}," +
|
|
|
|
|
|
$"{cs.MinPrice}," +
|
|
|
|
|
|
$"{cs.MaxPrice}," +
|
|
|
|
|
|
$"{cs.IsPrivate}," +
|
|
|
|
|
|
$"{cs.IsCommercial}" +
|
|
|
|
|
|
")" +
|
|
|
|
|
|
" RETURNING id;";
|
|
|
|
|
|
private static string SQL_UpdateCrawlSession(DB_CrawlSession cs)
|
|
|
|
|
|
=> "UPDATE " +
|
|
|
|
|
|
"crawl_session" +
|
|
|
|
|
|
" SET " +
|
|
|
|
|
|
$"keywords = '{cs.Keywords}'," +
|
|
|
|
|
|
$"location_text = '{cs.LocationText}'," +
|
|
|
|
|
|
$"category_id = {cs.CategoryID}," +
|
|
|
|
|
|
$"radius_km = {cs.RadiusKM}," +
|
|
|
|
|
|
$"min_price = {cs.MinPrice}," +
|
|
|
|
|
|
$"max_price = {cs.MaxPrice}," +
|
|
|
|
|
|
$"is_private = {cs.IsPrivate}," +
|
|
|
|
|
|
$"is_commercial = {cs.IsCommercial}" +
|
|
|
|
|
|
" WHERE " +
|
|
|
|
|
|
$"id = {cs.ID}" +
|
|
|
|
|
|
$" and user_id = {cs.UserID};";
|
|
|
|
|
|
private static string SQL_SeletCrawlSession(int userID) => $"SELECT " +
|
|
|
|
|
|
"id," +
|
|
|
|
|
|
"keywords," +
|
|
|
|
|
|
"location_text," +
|
|
|
|
|
|
"category_id," +
|
|
|
|
|
|
"radius_km," +
|
|
|
|
|
|
"min_price," +
|
|
|
|
|
|
"max_price," +
|
|
|
|
|
|
"is_private," +
|
|
|
|
|
|
"is_commercial" +
|
|
|
|
|
|
" FROM " +
|
|
|
|
|
|
"crawl_session" +
|
|
|
|
|
|
" WHERE " +
|
|
|
|
|
|
$"user_id = {userID};";
|
|
|
|
|
|
private static string SQL_SeletAllCrawlSessions() => $"SELECT " +
|
|
|
|
|
|
"id," +
|
|
|
|
|
|
"user_id," +
|
|
|
|
|
|
"keywords," +
|
|
|
|
|
|
"location_text," +
|
|
|
|
|
|
"category_id," +
|
|
|
|
|
|
"radius_km," +
|
|
|
|
|
|
"min_price," +
|
|
|
|
|
|
"max_price," +
|
|
|
|
|
|
"is_private," +
|
|
|
|
|
|
"is_commercial" +
|
|
|
|
|
|
" FROM " +
|
|
|
|
|
|
"crawl_session;";
|
|
|
|
|
|
private static string SQL_SeletAllCrawlSessionIDs(int userID)
|
|
|
|
|
|
=> $"SELECT " +
|
|
|
|
|
|
"id" +
|
|
|
|
|
|
" FROM " +
|
|
|
|
|
|
"crawl_session" +
|
|
|
|
|
|
" WHERE " +
|
|
|
|
|
|
$"user_id = {userID};";
|
|
|
|
|
|
private static string SQL_DeleteCrawlSession(int crawlSessionID)
|
|
|
|
|
|
=> $"DELETE FROM " +
|
|
|
|
|
|
"crawl_session" +
|
|
|
|
|
|
" WHERE " +
|
|
|
|
|
|
$"id = {crawlSessionID}";
|
|
|
|
|
|
private static string SQL_InsertInsertion(string href, int crawlSessionID, string name, int? postCode, string? locStr, decimal price, bool is_vb,
|
|
|
|
|
|
DateTime? date, bool isTopAd, bool isHighlight, bool isRequest)
|
|
|
|
|
|
=> $"INSERT INTO " +
|
|
|
|
|
|
"insertion" +
|
|
|
|
|
|
"(href, crawl_session_id, name, post_code, location_text, price, is_vb, date, " +
|
|
|
|
|
|
"is_topad, is_highlight, is_request)" +
|
|
|
|
|
|
" VALUES " +
|
|
|
|
|
|
"(" +
|
|
|
|
|
|
$"'{href}'," +
|
|
|
|
|
|
$"{crawlSessionID}," +
|
|
|
|
|
|
$"'{name}'," +
|
|
|
|
|
|
$"{Extensions.WriteNullableInt(postCode)}," +
|
|
|
|
|
|
$"{Extensions.WriteNullableString(locStr)}," +
|
|
|
|
|
|
$"{price}," +
|
|
|
|
|
|
$"{is_vb}," +
|
|
|
|
|
|
$"{Extensions.WriteNullableDateTime(date)}," +
|
|
|
|
|
|
$"{isTopAd}," +
|
|
|
|
|
|
$"{isHighlight}," +
|
|
|
|
|
|
$"{isRequest}" +
|
|
|
|
|
|
");";
|
|
|
|
|
|
private static string SQL_SelectInsertions(int crawlSessionID, int? limit)
|
|
|
|
|
|
=> $"SELECT " +
|
|
|
|
|
|
"href," +
|
|
|
|
|
|
"name," +
|
|
|
|
|
|
"post_code," +
|
|
|
|
|
|
"location_text," +
|
|
|
|
|
|
"price," +
|
|
|
|
|
|
"is_vb," +
|
|
|
|
|
|
"date," +
|
|
|
|
|
|
"is_topad," +
|
|
|
|
|
|
"is_highlight," +
|
|
|
|
|
|
"is_request" +
|
|
|
|
|
|
" FROM " +
|
|
|
|
|
|
"insertion" +
|
|
|
|
|
|
" WHERE " +
|
|
|
|
|
|
$"crawl_session_id = {crawlSessionID}" +
|
|
|
|
|
|
" and is_topad = false" +
|
|
|
|
|
|
" and is_highlight = false" + //TODO: these 3 booleans are untested
|
|
|
|
|
|
" and is_request = false" +
|
|
|
|
|
|
" ORDER BY id desc " +
|
|
|
|
|
|
$" LIMIT {limit?.ToString() ?? "null"}";
|
|
|
|
|
|
#endregion
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
