From a31c8b2ab8e308ccc21b0fcf8ae5bfc666486521 Mon Sep 17 00:00:00 2001 From: Matthew Wright Date: Sun, 24 Jan 2021 11:54:25 -0500 Subject: [PATCH 01/17] comment threads and replies max results 100 --- .../youtube/commentsuite/refresh/CommentThreadProducer.java | 2 +- .../io/mattw/youtube/commentsuite/refresh/ReplyProducer.java | 1 + .../io/mattw/youtube/commentsuite/fxml/MGMVRefreshModal.fxml | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/io/mattw/youtube/commentsuite/refresh/CommentThreadProducer.java b/src/main/java/io/mattw/youtube/commentsuite/refresh/CommentThreadProducer.java index 75fee15..0d56c0b 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/refresh/CommentThreadProducer.java +++ b/src/main/java/io/mattw/youtube/commentsuite/refresh/CommentThreadProducer.java @@ -67,7 +67,7 @@ private void produce() { .list("snippet") .setKey(FXMLSuite.getYouTubeApiKey()) .setVideoId(video.getId()) - .setMaxResults(50L) + .setMaxResults(100L) .setOrder(options.getCommentOrder().name()) .setPageToken(pageToken) .execute(); diff --git a/src/main/java/io/mattw/youtube/commentsuite/refresh/ReplyProducer.java b/src/main/java/io/mattw/youtube/commentsuite/refresh/ReplyProducer.java index 897fb0d..8dab6ad 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/refresh/ReplyProducer.java +++ b/src/main/java/io/mattw/youtube/commentsuite/refresh/ReplyProducer.java @@ -60,6 +60,7 @@ private void produce() { .setKey(FXMLSuite.getYouTubeApiKey()) .setParentId(tuple.getFirst()) .setPageToken(pageToken) + .setMaxResults(100L) .execute(); pageToken = response.getNextPageToken(); diff --git a/src/main/resources/io/mattw/youtube/commentsuite/fxml/MGMVRefreshModal.fxml b/src/main/resources/io/mattw/youtube/commentsuite/fxml/MGMVRefreshModal.fxml index d7af361..363c48f 100644 --- a/src/main/resources/io/mattw/youtube/commentsuite/fxml/MGMVRefreshModal.fxml +++ b/src/main/resources/io/mattw/youtube/commentsuite/fxml/MGMVRefreshModal.fxml @@ -20,7 +20,7 @@ - + @@ -29,7 +29,7 @@ - + Date: Sun, 24 Jan 2021 15:18:46 -0500 Subject: [PATCH 02/17] stop refresh on quota error --- pom.xml | 2 +- .../commentsuite/refresh/ChannelProducer.java | 2 +- .../refresh/CommentThreadProducer.java | 6 +++--- .../commentsuite/refresh/GroupRefresh.java | 15 +++++++++++++++ .../commentsuite/refresh/ReplyProducer.java | 4 ++-- .../commentsuite/refresh/VideoIdProducer.java | 3 ++- .../commentsuite/refresh/VideoProducer.java | 2 +- 7 files changed, 25 insertions(+), 9 deletions(-) diff --git a/pom.xml b/pom.xml index 63cc3cd..dbefe94 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ io.mattw.youtube youtube-comment-suite - 1.4.4 + 1.4.5 UTF-8 diff --git a/src/main/java/io/mattw/youtube/commentsuite/refresh/ChannelProducer.java b/src/main/java/io/mattw/youtube/commentsuite/refresh/ChannelProducer.java index 16751f2..79713f7 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/refresh/ChannelProducer.java +++ b/src/main/java/io/mattw/youtube/commentsuite/refresh/ChannelProducer.java @@ -107,7 +107,7 @@ private void produceChannels(final List channelIds) { logger.warn(e.getDetails().getMessage()); logger.warn("filter parameters [id={}]", String.join(",", channelIds)); } else { - e.printStackTrace(); + sendMessage(Level.ERROR, e, "Failed during query for channels"); } } catch (Exception e) { logger.error("Error on channel grab", e); diff --git a/src/main/java/io/mattw/youtube/commentsuite/refresh/CommentThreadProducer.java b/src/main/java/io/mattw/youtube/commentsuite/refresh/CommentThreadProducer.java index 0d56c0b..0882e97 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/refresh/CommentThreadProducer.java +++ b/src/main/java/io/mattw/youtube/commentsuite/refresh/CommentThreadProducer.java @@ -135,7 +135,7 @@ private void produce() { firstError.getReason(), video.getId()); - sendMessage(Level.WARN, e, message); + sendMessage(Level.ERROR, e, message); attempts++; } else if (ge.getStatusCode() == 403) { @@ -150,7 +150,7 @@ private void produce() { firstError.getReason(), video.getId()); - sendMessage(Level.WARN, e, message); + sendMessage(Level.ERROR, e, message); break; } @@ -161,7 +161,7 @@ private void produce() { e.getClass().getSimpleName(), video.getId()); - sendMessage(Level.WARN, e, message); + sendMessage(Level.ERROR, e, message); attempts++; } } diff --git a/src/main/java/io/mattw/youtube/commentsuite/refresh/GroupRefresh.java b/src/main/java/io/mattw/youtube/commentsuite/refresh/GroupRefresh.java index cd23f18..ab3ab35 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/refresh/GroupRefresh.java +++ b/src/main/java/io/mattw/youtube/commentsuite/refresh/GroupRefresh.java @@ -1,5 +1,7 @@ package io.mattw.youtube.commentsuite.refresh; +import com.google.api.client.googleapis.json.GoogleJsonError; +import com.google.api.client.googleapis.json.GoogleJsonResponseException; import io.mattw.youtube.commentsuite.FXMLSuite; import io.mattw.youtube.commentsuite.db.*; import io.mattw.youtube.commentsuite.util.ElapsedTime; @@ -19,6 +21,7 @@ import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.stream.Stream; import static javafx.application.Platform.runLater; @@ -185,6 +188,18 @@ private void postMessage(final Level level, final Throwable error, final String hardShutdown(); } + if (error instanceof GoogleJsonResponseException) { + final GoogleJsonResponseException googleError = (GoogleJsonResponseException) error; + final List errorInfos = googleError.getDetails().getErrors(); + if (googleError.getStatusCode() == 403 && errorInfos.stream() + .map(GoogleJsonError.ErrorInfo::getReason) + .anyMatch("quotaExceeded"::equals)) { + endedOnError = true; + hardShutdown(); + runLater(() -> errorList.add(0, String.format("%s - %s", time, googleError))); + } + } + if (message != null) { runLater(() -> errorList.add(0, String.format("%s - %s", time, message))); } diff --git a/src/main/java/io/mattw/youtube/commentsuite/refresh/ReplyProducer.java b/src/main/java/io/mattw/youtube/commentsuite/refresh/ReplyProducer.java index 8dab6ad..9d9e0ef 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/refresh/ReplyProducer.java +++ b/src/main/java/io/mattw/youtube/commentsuite/refresh/ReplyProducer.java @@ -9,6 +9,7 @@ import io.mattw.youtube.commentsuite.util.ExecutorGroup; import io.mattw.youtube.commentsuite.util.StringTuple; import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -82,8 +83,7 @@ private void produce() { awaitMillis(50); } while (pageToken != null && page++ < replyPages.getPageCount() && !isHardShutdown()); } catch (IOException e) { - e.printStackTrace(); - logger.error("Couldn't grab commentThread[id={}]", tuple.getFirst(), e); + sendMessage(Level.ERROR, e, String.format("Couldn't grab commentThread[id={}]", tuple.getFirst())); } addProcessed(1); diff --git a/src/main/java/io/mattw/youtube/commentsuite/refresh/VideoIdProducer.java b/src/main/java/io/mattw/youtube/commentsuite/refresh/VideoIdProducer.java index 00573fd..e73a68b 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/refresh/VideoIdProducer.java +++ b/src/main/java/io/mattw/youtube/commentsuite/refresh/VideoIdProducer.java @@ -8,6 +8,7 @@ import io.mattw.youtube.commentsuite.db.GroupItemVideo; import io.mattw.youtube.commentsuite.db.YType; import io.mattw.youtube.commentsuite.util.ExecutorGroup; +import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -59,7 +60,7 @@ private void produce() { updateGroupItem(item); } catch (IOException | SQLException e) { - logger.debug("Failed GroupItem {}", item, e); + sendMessage(Level.ERROR, e, String.format("Failed GroupItem %s", item)); } addProcessed(1); diff --git a/src/main/java/io/mattw/youtube/commentsuite/refresh/VideoProducer.java b/src/main/java/io/mattw/youtube/commentsuite/refresh/VideoProducer.java index 46b268c..49256ff 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/refresh/VideoProducer.java +++ b/src/main/java/io/mattw/youtube/commentsuite/refresh/VideoProducer.java @@ -68,7 +68,7 @@ private void produce() { queryAndInsert(videoIds); addProcessed(videoIds.size()); } catch (IOException | SQLException e) { - logger.error(e); + sendMessage(Level.ERROR, e, "Failed to query for videos"); } logger.debug("Ending VideoProducer"); From 6419dfff9d79c5486bacf9281fe601854e70eb47 Mon Sep 17 00:00:00 2001 From: Matthew Wright Date: Mon, 25 Jan 2021 10:40:40 -0500 Subject: [PATCH 03/17] move cleanable and exportable to package used --- .../java/io/mattw/youtube/commentsuite/db/CommentQuery.java | 1 - .../java/io/mattw/youtube/commentsuite/{ => db}/Exportable.java | 2 +- .../java/io/mattw/youtube/commentsuite/db/YouTubeComment.java | 1 - .../java/io/mattw/youtube/commentsuite/db/YouTubeVideo.java | 1 - .../io/mattw/youtube/commentsuite/{ => fxml}/Cleanable.java | 2 +- .../io/mattw/youtube/commentsuite/fxml/MGCreateGroupModal.java | 1 - .../io/mattw/youtube/commentsuite/fxml/MGMVAddItemModal.java | 1 - .../io/mattw/youtube/commentsuite/fxml/MGMVRemoveAllModal.java | 1 - .../youtube/commentsuite/fxml/MGMVRemoveSelectedModal.java | 1 - .../java/io/mattw/youtube/commentsuite/fxml/SCExportModal.java | 1 - .../io/mattw/youtube/commentsuite/fxml/SCVideoSelectModal.java | 1 - .../io/mattw/youtube/commentsuite/fxml/SYAddToGroupModal.java | 1 - 12 files changed, 2 insertions(+), 12 deletions(-) rename src/main/java/io/mattw/youtube/commentsuite/{ => db}/Exportable.java (93%) rename src/main/java/io/mattw/youtube/commentsuite/{ => fxml}/Cleanable.java (86%) diff --git a/src/main/java/io/mattw/youtube/commentsuite/db/CommentQuery.java b/src/main/java/io/mattw/youtube/commentsuite/db/CommentQuery.java index dec4ac5..10724cf 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/db/CommentQuery.java +++ b/src/main/java/io/mattw/youtube/commentsuite/db/CommentQuery.java @@ -1,6 +1,5 @@ package io.mattw.youtube.commentsuite.db; -import io.mattw.youtube.commentsuite.Exportable; import io.mattw.youtube.commentsuite.util.DateUtils; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; diff --git a/src/main/java/io/mattw/youtube/commentsuite/Exportable.java b/src/main/java/io/mattw/youtube/commentsuite/db/Exportable.java similarity index 93% rename from src/main/java/io/mattw/youtube/commentsuite/Exportable.java rename to src/main/java/io/mattw/youtube/commentsuite/db/Exportable.java index 5437038..f63cfed 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/Exportable.java +++ b/src/main/java/io/mattw/youtube/commentsuite/db/Exportable.java @@ -1,4 +1,4 @@ -package io.mattw.youtube.commentsuite; +package io.mattw.youtube.commentsuite.db; /** * To be used on classes that may be exported to JSON files as part of the export process. diff --git a/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeComment.java b/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeComment.java index 520fc55..0b2eb41 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeComment.java +++ b/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeComment.java @@ -4,7 +4,6 @@ import com.google.api.services.youtube.model.CommentSnippet; import com.google.api.services.youtube.model.CommentThread; import com.google.api.services.youtube.model.CommentThreadSnippet; -import io.mattw.youtube.commentsuite.Exportable; import io.mattw.youtube.commentsuite.FXMLSuite; import io.mattw.youtube.commentsuite.util.DateUtils; import org.apache.commons.lang3.builder.ReflectionToStringBuilder; diff --git a/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeVideo.java b/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeVideo.java index 5caf3db..427c4ab 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeVideo.java +++ b/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeVideo.java @@ -3,7 +3,6 @@ import com.google.api.services.youtube.model.Video; import com.google.api.services.youtube.model.VideoSnippet; import com.google.api.services.youtube.model.VideoStatistics; -import io.mattw.youtube.commentsuite.Exportable; import io.mattw.youtube.commentsuite.util.DateUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/src/main/java/io/mattw/youtube/commentsuite/Cleanable.java b/src/main/java/io/mattw/youtube/commentsuite/fxml/Cleanable.java similarity index 86% rename from src/main/java/io/mattw/youtube/commentsuite/Cleanable.java rename to src/main/java/io/mattw/youtube/commentsuite/fxml/Cleanable.java index 49f2905..d7ae882 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/Cleanable.java +++ b/src/main/java/io/mattw/youtube/commentsuite/fxml/Cleanable.java @@ -1,4 +1,4 @@ -package io.mattw.youtube.commentsuite; +package io.mattw.youtube.commentsuite.fxml; /** * For use in instances where UI components can "clean up" and reset their state instead of creating an diff --git a/src/main/java/io/mattw/youtube/commentsuite/fxml/MGCreateGroupModal.java b/src/main/java/io/mattw/youtube/commentsuite/fxml/MGCreateGroupModal.java index 8af1500..1a91ded 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/fxml/MGCreateGroupModal.java +++ b/src/main/java/io/mattw/youtube/commentsuite/fxml/MGCreateGroupModal.java @@ -1,6 +1,5 @@ package io.mattw.youtube.commentsuite.fxml; -import io.mattw.youtube.commentsuite.Cleanable; import javafx.fxml.FXML; import javafx.fxml.FXMLLoader; import javafx.scene.control.Button; diff --git a/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVAddItemModal.java b/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVAddItemModal.java index f752fac..480c390 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVAddItemModal.java +++ b/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVAddItemModal.java @@ -1,6 +1,5 @@ package io.mattw.youtube.commentsuite.fxml; -import io.mattw.youtube.commentsuite.Cleanable; import io.mattw.youtube.commentsuite.ConfigData; import io.mattw.youtube.commentsuite.ConfigFile; import io.mattw.youtube.commentsuite.FXMLSuite; diff --git a/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVRemoveAllModal.java b/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVRemoveAllModal.java index bf8dbe1..48b91ed 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVRemoveAllModal.java +++ b/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVRemoveAllModal.java @@ -1,6 +1,5 @@ package io.mattw.youtube.commentsuite.fxml; -import io.mattw.youtube.commentsuite.Cleanable; import io.mattw.youtube.commentsuite.FXMLSuite; import io.mattw.youtube.commentsuite.db.CommentDatabase; import io.mattw.youtube.commentsuite.db.Group; diff --git a/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVRemoveSelectedModal.java b/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVRemoveSelectedModal.java index e341927..e9d2615 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVRemoveSelectedModal.java +++ b/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVRemoveSelectedModal.java @@ -1,6 +1,5 @@ package io.mattw.youtube.commentsuite.fxml; -import io.mattw.youtube.commentsuite.Cleanable; import io.mattw.youtube.commentsuite.FXMLSuite; import io.mattw.youtube.commentsuite.db.CommentDatabase; import io.mattw.youtube.commentsuite.db.Group; diff --git a/src/main/java/io/mattw/youtube/commentsuite/fxml/SCExportModal.java b/src/main/java/io/mattw/youtube/commentsuite/fxml/SCExportModal.java index 22f68dc..2b37fba 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/fxml/SCExportModal.java +++ b/src/main/java/io/mattw/youtube/commentsuite/fxml/SCExportModal.java @@ -3,7 +3,6 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonParser; -import io.mattw.youtube.commentsuite.Cleanable; import io.mattw.youtube.commentsuite.FXMLSuite; import io.mattw.youtube.commentsuite.ImageCache; import io.mattw.youtube.commentsuite.db.CommentDatabase; diff --git a/src/main/java/io/mattw/youtube/commentsuite/fxml/SCVideoSelectModal.java b/src/main/java/io/mattw/youtube/commentsuite/fxml/SCVideoSelectModal.java index 1c12515..9562609 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/fxml/SCVideoSelectModal.java +++ b/src/main/java/io/mattw/youtube/commentsuite/fxml/SCVideoSelectModal.java @@ -1,6 +1,5 @@ package io.mattw.youtube.commentsuite.fxml; -import io.mattw.youtube.commentsuite.Cleanable; import io.mattw.youtube.commentsuite.FXMLSuite; import io.mattw.youtube.commentsuite.ImageLoader; import io.mattw.youtube.commentsuite.db.CommentDatabase; diff --git a/src/main/java/io/mattw/youtube/commentsuite/fxml/SYAddToGroupModal.java b/src/main/java/io/mattw/youtube/commentsuite/fxml/SYAddToGroupModal.java index 1d85ceb..5c47a53 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/fxml/SYAddToGroupModal.java +++ b/src/main/java/io/mattw/youtube/commentsuite/fxml/SYAddToGroupModal.java @@ -1,7 +1,6 @@ package io.mattw.youtube.commentsuite.fxml; import com.google.common.eventbus.Subscribe; -import io.mattw.youtube.commentsuite.Cleanable; import io.mattw.youtube.commentsuite.FXMLSuite; import io.mattw.youtube.commentsuite.db.CommentDatabase; import io.mattw.youtube.commentsuite.db.Group; From 93c0538ded6179a312cc339b3412238fe99054c9 Mon Sep 17 00:00:00 2001 From: Matthew Wright Date: Wed, 27 Jan 2021 21:07:52 -0500 Subject: [PATCH 04/17] moderated comments; bunch of refactoring and clean up --- BUILD.md | 2 +- pom.xml | 2 +- .../{FXMLSuite.java => CommentSuite.java} | 14 +- .../youtube/commentsuite/ConfigData.java | 142 +++++++------- .../youtube/commentsuite/ConfigFile.java | 17 +- .../youtube/commentsuite/ImageCache.java | 24 +-- .../youtube/commentsuite/ImageLoader.java | 5 +- .../youtube/commentsuite/LetterAvatar.java | 1 - .../ListViewEmptyCellFactory.java | 8 +- .../youtube/commentsuite/OAuth2Handler.java | 118 ----------- .../youtube/commentsuite/OAuth2Tokens.java | 32 --- .../youtube/commentsuite/YouTubeAccount.java | 119 ----------- .../commentsuite/db/CommentDatabase.java | 185 +++++++++++++----- .../youtube/commentsuite/db/CommentQuery.java | 20 +- .../mattw/youtube/commentsuite/db/Group.java | 3 - .../youtube/commentsuite/db/GroupItem.java | 64 +++--- .../commentsuite/db/GroupItemVideo.java | 10 +- .../youtube/commentsuite/db/GroupStats.java | 15 +- .../youtube/commentsuite/db/SQLLoader.java | 6 +- .../mattw/youtube/commentsuite/db/YType.java | 1 - .../commentsuite/db/YouTubeChannel.java | 3 - .../commentsuite/db/YouTubeComment.java | 49 ++++- .../commentsuite/db/YouTubeObject.java | 23 +-- .../youtube/commentsuite/db/YouTubeVideo.java | 3 - .../commentsuite/events/AccountAddEvent.java | 17 ++ .../events/AccountDeleteEvent.java | 17 ++ .../commentsuite/events/ReplyEvent.java | 17 ++ .../commentsuite/events/ShowMoreEvent.java | 17 ++ .../commentsuite/events/ViewTreeEvent.java | 17 ++ .../youtube/commentsuite/fxml/Cleanable.java | 1 - .../commentsuite/fxml/MGCreateGroupModal.java | 1 - .../commentsuite/fxml/MGMVAddItemModal.java | 20 +- .../fxml/MGMVDeleteGroupModal.java | 1 - .../commentsuite/fxml/MGMVGroupItemView.java | 5 +- .../commentsuite/fxml/MGMVRefreshModal.java | 134 ++++++++++--- .../commentsuite/fxml/MGMVRemoveAllModal.java | 11 +- .../fxml/MGMVRemoveSelectedModal.java | 11 +- .../fxml/MGMVYouTubeObjectItem.java | 15 +- .../mattw/youtube/commentsuite/fxml/Main.java | 1 - .../commentsuite/fxml/ManageGroups.java | 8 +- .../fxml/ManageGroupsManager.java | 94 ++++++--- .../commentsuite/fxml/OverlayModal.java | 9 +- .../commentsuite/fxml/SCExportModal.java | 11 +- .../commentsuite/fxml/SCExportProducer.java | 6 +- .../commentsuite/fxml/SCShowMoreModal.java | 104 +++++++--- .../commentsuite/fxml/SCVideoSelectModal.java | 20 +- .../commentsuite/fxml/SYAddToGroupModal.java | 17 +- .../commentsuite/fxml/SearchComments.java | 155 ++++++--------- .../fxml/SearchCommentsListItem.java | 99 +++++++--- .../commentsuite/fxml/SearchYouTube.java | 40 ++-- .../fxml/SearchYouTubeListItem.java | 12 +- .../youtube/commentsuite/fxml/Settings.java | 105 +++++----- .../fxml/SettingsAccountItemView.java | 14 +- .../{ => fxml}/YouTubeSearchItem.java | 5 +- .../commentsuite/oauth2/OAuth2Manager.java | 157 +++++++++++++++ .../commentsuite/oauth2/OAuth2Tokens.java | 50 +++++ .../commentsuite/oauth2/YouTubeAccount.java | 56 ++++++ .../commentsuite/refresh/ChannelConsumer.java | 4 +- .../commentsuite/refresh/ChannelProducer.java | 6 +- .../commentsuite/refresh/CommentConsumer.java | 30 ++- .../refresh/CommentThreadProducer.java | 112 +++++++++-- .../refresh/ConsumerMultiProducer.java | 36 ++++ .../commentsuite/refresh/GroupRefresh.java | 127 +++++++----- .../refresh/ModerationStatus.java | 52 +++++ .../refresh/RefreshInterface.java | 7 +- .../commentsuite/refresh/RefreshOptions.java | 24 ++- .../refresh/RefreshReplyPages.java | 53 ----- .../commentsuite/refresh/RefreshStyle.java | 30 ++- .../commentsuite/refresh/ReplyProducer.java | 8 +- .../refresh/UniqueVideoIdProducer.java | 4 +- .../commentsuite/refresh/VideoIdProducer.java | 10 +- .../commentsuite/refresh/VideoProducer.java | 23 ++- .../commentsuite/util/BrowserUtil.java | 1 - .../commentsuite/util/ClipboardUtil.java | 3 - .../youtube/commentsuite/util/DateUtils.java | 4 +- .../commentsuite/util/ElapsedTime.java | 1 - .../commentsuite/util/ExecutorGroup.java | 1 - .../youtube/commentsuite/util/FXUtils.java | 6 +- .../commentsuite/util/IpApiProvider.java | 3 - .../youtube/commentsuite/util/Location.java | 1 - .../commentsuite/util/LocationProvider.java | 3 - .../commentsuite/util/StringTuple.java | 2 +- .../youtube/commentsuite/util/Threads.java | 12 ++ .../commentsuite/util/UTF8UrlEncoder.java | 1 - src/main/resources/META-INF/MANIFEST.MF | 2 +- src/main/resources/SuiteStyles.css | 12 ++ .../commentsuite/db/sql/ddl_create_db.sql | 157 +++++++++------ .../commentsuite/db/sql/dml_clean_db.sql | 2 + .../db/sql/dml_delete_comment.sql | 1 + .../db/sql/dml_delete_moderated_comment.sql | 1 + .../dml_insert_ignore_moderated_comments.sql | 3 + .../sql/dql_does_moderated_comment_exist.sql | 1 + .../db/sql/dql_get_comment_tree.sql | 10 +- .../sql/dql_get_moderated_comment_stats.sql | 7 + .../commentsuite/fxml/MGMVRefreshModal.fxml | 36 ++-- .../fxml/ManageGroupsManager.fxml | 58 +----- .../fxml/SearchCommentsListItem.fxml | 3 + .../youtube/commentsuite/fxml/Settings.fxml | 2 + .../commentsuite/fxml/SearchCommentsTest.java | 20 +- 99 files changed, 1783 insertions(+), 1209 deletions(-) rename src/main/java/io/mattw/youtube/commentsuite/{FXMLSuite.java => CommentSuite.java} (90%) delete mode 100644 src/main/java/io/mattw/youtube/commentsuite/OAuth2Handler.java delete mode 100644 src/main/java/io/mattw/youtube/commentsuite/OAuth2Tokens.java delete mode 100644 src/main/java/io/mattw/youtube/commentsuite/YouTubeAccount.java create mode 100644 src/main/java/io/mattw/youtube/commentsuite/events/AccountAddEvent.java create mode 100644 src/main/java/io/mattw/youtube/commentsuite/events/AccountDeleteEvent.java create mode 100644 src/main/java/io/mattw/youtube/commentsuite/events/ReplyEvent.java create mode 100644 src/main/java/io/mattw/youtube/commentsuite/events/ShowMoreEvent.java create mode 100644 src/main/java/io/mattw/youtube/commentsuite/events/ViewTreeEvent.java rename src/main/java/io/mattw/youtube/commentsuite/{ => fxml}/YouTubeSearchItem.java (85%) create mode 100644 src/main/java/io/mattw/youtube/commentsuite/oauth2/OAuth2Manager.java create mode 100644 src/main/java/io/mattw/youtube/commentsuite/oauth2/OAuth2Tokens.java create mode 100644 src/main/java/io/mattw/youtube/commentsuite/oauth2/YouTubeAccount.java create mode 100644 src/main/java/io/mattw/youtube/commentsuite/refresh/ModerationStatus.java delete mode 100644 src/main/java/io/mattw/youtube/commentsuite/refresh/RefreshReplyPages.java create mode 100644 src/main/java/io/mattw/youtube/commentsuite/util/Threads.java create mode 100644 src/main/resources/io/mattw/youtube/commentsuite/db/sql/dml_delete_comment.sql create mode 100644 src/main/resources/io/mattw/youtube/commentsuite/db/sql/dml_delete_moderated_comment.sql create mode 100644 src/main/resources/io/mattw/youtube/commentsuite/db/sql/dml_insert_ignore_moderated_comments.sql create mode 100644 src/main/resources/io/mattw/youtube/commentsuite/db/sql/dql_does_moderated_comment_exist.sql create mode 100644 src/main/resources/io/mattw/youtube/commentsuite/db/sql/dql_get_moderated_comment_stats.sql diff --git a/BUILD.md b/BUILD.md index 8f04dab..64bde27 100644 --- a/BUILD.md +++ b/BUILD.md @@ -40,7 +40,7 @@ Maven will also insert these values into `Settings.fxml` during the build. There are two ways to run this application from your IDE -1. **From in IDE** Right click on file and run `src/main/java/io.mattw.youtube.commentsuite/FXMLSuite.java` +1. **From in IDE** Right click on file and run `src/main/java/io.mattw.youtube.commentsuite/CommentSuite.java` 2. **From package** Double click the jar at `target/package/youtube-comment-suite-#.#.#.jar` *Note: My YouTube API key provided in the application is not restricted at all given it isn't a website. diff --git a/pom.xml b/pom.xml index dbefe94..9433ae9 100644 --- a/pom.xml +++ b/pom.xml @@ -44,7 +44,7 @@ true lib/ - io.mattw.youtube.commentsuite.FXMLSuite + io.mattw.youtube.commentsuite.CommentSuite diff --git a/src/main/java/io/mattw/youtube/commentsuite/FXMLSuite.java b/src/main/java/io/mattw/youtube/commentsuite/CommentSuite.java similarity index 90% rename from src/main/java/io/mattw/youtube/commentsuite/FXMLSuite.java rename to src/main/java/io/mattw/youtube/commentsuite/CommentSuite.java index 26b9317..0ce1ef7 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/FXMLSuite.java +++ b/src/main/java/io/mattw/youtube/commentsuite/CommentSuite.java @@ -5,6 +5,7 @@ import com.google.api.services.youtube.YouTube; import com.google.common.eventbus.EventBus; import io.mattw.youtube.commentsuite.db.CommentDatabase; +import io.mattw.youtube.commentsuite.oauth2.OAuth2Manager; import io.mattw.youtube.commentsuite.util.IpApiProvider; import io.mattw.youtube.commentsuite.util.Location; import javafx.application.Application; @@ -23,31 +24,28 @@ /** * Application Window * - * @author mattwright324 */ -public class FXMLSuite extends Application { +public class CommentSuite extends Application { private static final Logger logger = LogManager.getLogger(); private static final ConfigFile config = new ConfigFile<>("commentsuite.json", new ConfigData()); private static final EventBus eventBus = new EventBus(); private static final Location location = new Location<>(new IpApiProvider(), IpApiProvider.Location.class); - private static final OAuth2Handler oauth2 = new OAuth2Handler("972416191049-htqcmg31u2t7hbd1ncen2e2jsg68cnqn.apps.googleusercontent.com", "QuTdoA-KArupKMWwDrrxOcoS", "urn:ietf:wg:oauth:2.0:oob"); private static CommentDatabase database; private static YouTube youTube; private static String youTubeApiKey; + private static OAuth2Manager oauth2Manager; public void start(final Stage stage) { try { youTube = new YouTube.Builder(GoogleNetHttpTransport.newTrustedTransport(), JacksonFactory.getDefaultInstance(), null) .setApplicationName("youtube-comment-suite") .build(); - - // System.setProperty("glass.win.uiScale", "100%"); youTubeApiKey = config.getDataObject().getYoutubeApiKey(); - database = new CommentDatabase("commentsuite.sqlite3"); + oauth2Manager = new OAuth2Manager(); final Parent parent = FXMLLoader.load(getClass().getResource("/io/mattw/youtube/commentsuite/fxml/Main.fxml")); @@ -90,8 +88,8 @@ public static Location getLocation() { return location; } - public static OAuth2Handler getOauth2() { - return oauth2; + public static OAuth2Manager getOauth2Manager() { + return oauth2Manager; } public static CommentDatabase getDatabase() { diff --git a/src/main/java/io/mattw/youtube/commentsuite/ConfigData.java b/src/main/java/io/mattw/youtube/commentsuite/ConfigData.java index 5e3a6b6..58ad0fb 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/ConfigData.java +++ b/src/main/java/io/mattw/youtube/commentsuite/ConfigData.java @@ -1,90 +1,93 @@ package io.mattw.youtube.commentsuite; +import io.mattw.youtube.commentsuite.events.AccountAddEvent; +import io.mattw.youtube.commentsuite.events.AccountDeleteEvent; +import io.mattw.youtube.commentsuite.oauth2.YouTubeAccount; import io.mattw.youtube.commentsuite.refresh.RefreshOptions; -import javafx.beans.property.ReadOnlyIntegerProperty; -import javafx.beans.property.SimpleIntegerProperty; import java.io.Serializable; import java.util.ArrayList; import java.util.List; -import static javafx.application.Platform.runLater; - -/** - * @author mattwright324 - */ public class ConfigData implements Serializable { - private transient String defaultApiKey = "AIzaSyD9SzQFnmOn08ESZC-7gIhnHWVn0asfrKQ"; - private transient SimpleIntegerProperty accountListChanged = new SimpleIntegerProperty(0); - + public static final transient String DEFAULT_API_KEY = "AIzaSyD9SzQFnmOn08ESZC-7gIhnHWVn0asfrKQ"; public static final transient String FAST_GROUP_ADD_THUMB_PLACEHOLDER = "~"; - private boolean autoLoadStats = true; - private boolean prefixReplies = true; + private List accounts = new ArrayList<>(); private boolean archiveThumbs = false; - private boolean fastGroupAdd = false; + private boolean autoLoadStats = true; private boolean customApiKey = false; + private boolean fastGroupAdd = false; private boolean filterDuplicatesOnCopy = true; - private List accounts = new ArrayList<>(); - private String youtubeApiKey = defaultApiKey; + private boolean grabHeldForReview = false; + private boolean grabLikelySpam = false; + private boolean prefixReplies = true; private RefreshOptions refreshOptions = new RefreshOptions(); + private String youtubeApiKey = DEFAULT_API_KEY; - public ConfigData() { - // empty constructor + public List getAccounts() { + return accounts; } - public String getDefaultApiKey() { - return defaultApiKey; + public void setAccounts(List accounts) { + this.accounts = accounts; } - public String getYoutubeApiKey() { - return customApiKey ? youtubeApiKey : defaultApiKey; + public boolean isArchiveThumbs() { + return archiveThumbs; } - public void setYoutubeApiKey(String apiKey) { - this.youtubeApiKey = apiKey; + public void setArchiveThumbs(boolean archiveThumbs) { + this.archiveThumbs = archiveThumbs; } - public List getAccounts() { - return accounts; + public boolean isAutoLoadStats() { + return autoLoadStats; } - public void refreshAccounts() { - accounts.forEach(YouTubeAccount::updateData); + public void setAutoLoadStats(boolean autoLoadStats) { + this.autoLoadStats = autoLoadStats; } - public void addAccount(YouTubeAccount account) { - if (accounts.stream().noneMatch(ac -> ac.getChannelId().equals(account.getChannelId()))) { - accounts.add(account); - triggerAccountListChanged(); - } + public boolean isCustomApiKey() { + return customApiKey; } - public void removeAccount(YouTubeAccount account) { - if (accounts.removeIf(acc -> acc.getChannelId() != null && acc.getChannelId().equals(account.getChannelId()))) { - triggerAccountListChanged(); - } + public void setCustomApiKey(boolean customApiKey) { + this.customApiKey = customApiKey; } - public boolean isSignedIn(String channelId) { - return accounts.stream().anyMatch(acc -> channelId.equals(acc.getChannelId())); + public boolean isFastGroupAdd() { + return fastGroupAdd; } - public ReadOnlyIntegerProperty accountListChangedProperty() { - return accountListChanged; + public void setFastGroupAdd(boolean fastGroupAdd) { + this.fastGroupAdd = fastGroupAdd; } - public void triggerAccountListChanged() { - runLater(() -> accountListChanged.setValue(accountListChanged.getValue() + 1)); + public boolean isFilterDuplicatesOnCopy() { + return filterDuplicatesOnCopy; } - public boolean isAutoLoadStats() { - return autoLoadStats; + public void setFilterDuplicatesOnCopy(boolean filterDuplicatesOnCopy) { + this.filterDuplicatesOnCopy = filterDuplicatesOnCopy; } - public void setAutoLoadStats(boolean autoLoadStats) { - this.autoLoadStats = autoLoadStats; + public boolean isGrabHeldForReview() { + return grabHeldForReview; + } + + public void setGrabHeldForReview(boolean grabHeldForReview) { + this.grabHeldForReview = grabHeldForReview; + } + + public boolean isGrabLikelySpam() { + return grabLikelySpam; + } + + public void setGrabLikelySpam(boolean grabLikelySpam) { + this.grabLikelySpam = grabLikelySpam; } public boolean isPrefixReplies() { @@ -95,43 +98,44 @@ public void setPrefixReplies(boolean prefixReplies) { this.prefixReplies = prefixReplies; } - public boolean isArchiveThumbs() { - return archiveThumbs; - } - - public void setArchiveThumbs(boolean archiveThumbs) { - this.archiveThumbs = archiveThumbs; + public RefreshOptions getRefreshOptions() { + return refreshOptions; } - public boolean isFastGroupAdd() { - return fastGroupAdd; + public void setRefreshOptions(RefreshOptions refreshOptions) { + this.refreshOptions = refreshOptions; } - public void setFastGroupAdd(boolean fastGroupAdd) { - this.fastGroupAdd = fastGroupAdd; + public String getYoutubeApiKey() { + return youtubeApiKey; } - public boolean isCustomApiKey() { - return customApiKey; + public void setYoutubeApiKey(String youtubeApiKey) { + this.youtubeApiKey = youtubeApiKey; } - public void setCustomApiKey(boolean customApiKey) { - this.customApiKey = customApiKey; + public void addAccount(final YouTubeAccount account) { + if (accounts.stream().noneMatch(ac -> ac.getChannelId().equals(account.getChannelId()))) { + accounts.add(account); + CommentSuite.getEventBus().post(new AccountAddEvent(account)); + } } - public boolean isFilterDuplicatesOnCopy() { - return filterDuplicatesOnCopy; + public void removeAccount(final YouTubeAccount account) { + if (accounts.removeIf(acc -> acc.getChannelId() != null && acc.getChannelId().equals(account.getChannelId()))) { + CommentSuite.getEventBus().post(new AccountDeleteEvent(account)); + } } - public void setFilterDuplicatesOnCopy(boolean filterDuplicatesOnCopy) { - this.filterDuplicatesOnCopy = filterDuplicatesOnCopy; + public boolean isSignedIn(final String channelId) { + return accounts.stream().anyMatch(acc -> channelId.equals(acc.getChannelId())); } - public RefreshOptions getRefreshOptions() { - return refreshOptions; + public YouTubeAccount getAccount(final String channelId) { + return accounts.stream() + .filter(acc -> channelId.equals(acc.getChannelId())) + .findFirst() + .orElse(null); } - public void setRefreshOptions(RefreshOptions refreshOptions) { - this.refreshOptions = refreshOptions; - } } diff --git a/src/main/java/io/mattw/youtube/commentsuite/ConfigFile.java b/src/main/java/io/mattw/youtube/commentsuite/ConfigFile.java index 11a49a3..7c95b91 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/ConfigFile.java +++ b/src/main/java/io/mattw/youtube/commentsuite/ConfigFile.java @@ -13,19 +13,18 @@ * Can take any object as long as it has fields that Gson is configured to read. * * @param data object JSON (de)serialized - * @author mattwright324 */ public class ConfigFile { private static final Logger logger = LogManager.getLogger(); - private Gson gson = new Gson(); + private final Gson gson = new Gson(); - private T defaultObject; + private final File file; + private final T defaultObject; private T dataObject; - private File file; - public ConfigFile(String fileName, T defaultObject) { + public ConfigFile(final String fileName, final T defaultObject) { logger.debug("Initialize ConfigFile<{}> [fileName={}]", defaultObject.getClass().getSimpleName(), fileName); this.defaultObject = defaultObject; this.dataObject = defaultObject; @@ -39,8 +38,8 @@ public ConfigFile(String fileName, T defaultObject) { public void load() { logger.debug("Loading Config File"); - try (FileReader fr = new FileReader(file); - BufferedReader br = new BufferedReader(fr)) { + try (final FileReader fr = new FileReader(file); + final BufferedReader br = new BufferedReader(fr)) { String line; StringBuilder data = new StringBuilder(); while ((line = br.readLine()) != null) { @@ -60,8 +59,8 @@ public void load() { public void save() { logger.debug("Saving Config File"); - try (FileOutputStream fos = new FileOutputStream(file); - OutputStreamWriter writer = new OutputStreamWriter(fos, StandardCharsets.UTF_8)) { + try (final FileOutputStream fos = new FileOutputStream(file); + final OutputStreamWriter writer = new OutputStreamWriter(fos, StandardCharsets.UTF_8)) { writer.write(gson.toJson(dataObject)); } catch (Exception e) { logger.error(e); diff --git a/src/main/java/io/mattw/youtube/commentsuite/ImageCache.java b/src/main/java/io/mattw/youtube/commentsuite/ImageCache.java index c5a68c7..432873d 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/ImageCache.java +++ b/src/main/java/io/mattw/youtube/commentsuite/ImageCache.java @@ -3,6 +3,7 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import io.mattw.youtube.commentsuite.db.YouTubeObject; +import io.mattw.youtube.commentsuite.oauth2.YouTubeAccount; import javafx.scene.image.Image; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -17,7 +18,6 @@ /** * Cache for images and letter avatars. * - * @author mattwright324 */ public interface ImageCache { @@ -31,11 +31,11 @@ public interface ImageCache { .expireAfterAccess(5, TimeUnit.MINUTES) .build(); - static Image toLetterAvatar(YouTubeObject object) { + static Image toLetterAvatar(final YouTubeObject object) { return toLetterAvatar(object.getTitle()); } - static Image toLetterAvatar(String s) { + static Image toLetterAvatar(final String s) { if (s == null || s.isEmpty()) { return toLetterAvatar(" "); } else { @@ -43,7 +43,7 @@ static Image toLetterAvatar(String s) { } } - static Image toLetterAvatar(char letter) { + static Image toLetterAvatar(final char letter) { Image image = thumbCache.getIfPresent(letter); if (image == null) { image = new LetterAvatar(letter); @@ -52,9 +52,9 @@ static Image toLetterAvatar(char letter) { return image; } - static Image findOrGetImage(String id, String imageUrl) { - ConfigFile config = FXMLSuite.getConfig(); - ConfigData configData = config.getDataObject(); + static Image findOrGetImage(final String id, final String imageUrl) { + final ConfigFile config = CommentSuite.getConfig(); + final ConfigData configData = config.getDataObject(); if (ConfigData.FAST_GROUP_ADD_THUMB_PLACEHOLDER.equals(imageUrl)) { return null; @@ -62,13 +62,13 @@ static Image findOrGetImage(String id, String imageUrl) { Image image = thumbCache.getIfPresent(id); if (image == null) { - File thumbFile = new File(thumbsDir, String.format("%s.%s", id, thumbFormat)); + final File thumbFile = new File(thumbsDir, String.format("%s.%s", id, thumbFormat)); if (configData.isArchiveThumbs() && !thumbFile.exists()) { thumbsDir.mkdir(); logger.debug("Archiving [id={}]", id); try { - BufferedImage bufferedImage = ImageIO.read(new URL(imageUrl)); + final BufferedImage bufferedImage = ImageIO.read(new URL(imageUrl)); thumbFile.createNewFile(); @@ -90,15 +90,15 @@ static Image findOrGetImage(String id, String imageUrl) { return image; } - static Image findOrGetImage(YouTubeObject object) { + static Image findOrGetImage(final YouTubeObject object) { return findOrGetImage(object.getId(), object.getThumbUrl()); } - static Image findOrGetImage(YouTubeAccount account) { + static Image findOrGetImage(final YouTubeAccount account) { return findOrGetImage(account.getChannelId(), account.getThumbUrl()); } - static boolean hasImageCached(YouTubeObject object) { + static boolean hasImageCached(final YouTubeObject object) { return thumbCache.getIfPresent(object.getId()) != null; } } diff --git a/src/main/java/io/mattw/youtube/commentsuite/ImageLoader.java b/src/main/java/io/mattw/youtube/commentsuite/ImageLoader.java index bca8a41..da5c602 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/ImageLoader.java +++ b/src/main/java/io/mattw/youtube/commentsuite/ImageLoader.java @@ -4,9 +4,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -/** - * @author mattwright324 - */ public enum ImageLoader { ANGLE_DOUBLE_LEFT("angle-double-left.png"), ANGLE_DOUBLE_RIGHT("angle-double-right.png"), @@ -45,7 +42,7 @@ public enum ImageLoader { private final Logger logger = LogManager.getLogger(); - private String basePath = "/io/mattw/youtube/commentsuite/img"; + private final String basePath = "/io/mattw/youtube/commentsuite/img"; private Image image; ImageLoader(String fileName) { diff --git a/src/main/java/io/mattw/youtube/commentsuite/LetterAvatar.java b/src/main/java/io/mattw/youtube/commentsuite/LetterAvatar.java index 1aec414..3195e87 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/LetterAvatar.java +++ b/src/main/java/io/mattw/youtube/commentsuite/LetterAvatar.java @@ -14,7 +14,6 @@ /** * Generates images with a character to mimic the style of default YouTube generated letter-avatars. * - * @author mattwright324 */ public class LetterAvatar extends WritableImage { diff --git a/src/main/java/io/mattw/youtube/commentsuite/ListViewEmptyCellFactory.java b/src/main/java/io/mattw/youtube/commentsuite/ListViewEmptyCellFactory.java index 8bd17f0..04ea4fd 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/ListViewEmptyCellFactory.java +++ b/src/main/java/io/mattw/youtube/commentsuite/ListViewEmptyCellFactory.java @@ -10,8 +10,6 @@ * Option to display tooltip on all cells to find preferred dummy cell height. *

* https://stackoverflow.com/a/46261347/2650847 - * - * @author mattwright324 */ public class ListViewEmptyCellFactory extends ListCell { @@ -23,17 +21,17 @@ public ListViewEmptyCellFactory() { tool.textProperty().bind(heightProperty().asString()); } - public ListViewEmptyCellFactory(double height) { + public ListViewEmptyCellFactory(final double height) { this(height, false); } - public ListViewEmptyCellFactory(double height, boolean tooltipHeight) { + public ListViewEmptyCellFactory(final double height, final boolean tooltipHeight) { this(); this.height = height; this.tooltipHeight = tooltipHeight; } - protected void updateItem(T item, boolean empty) { + protected void updateItem(final T item, final boolean empty) { super.updateItem(item, empty); setTooltip(tooltipHeight ? tool : null); if (empty) { diff --git a/src/main/java/io/mattw/youtube/commentsuite/OAuth2Handler.java b/src/main/java/io/mattw/youtube/commentsuite/OAuth2Handler.java deleted file mode 100644 index f8aec3f..0000000 --- a/src/main/java/io/mattw/youtube/commentsuite/OAuth2Handler.java +++ /dev/null @@ -1,118 +0,0 @@ -package io.mattw.youtube.commentsuite; - -import com.google.api.services.youtube.model.Comment; -import com.google.api.services.youtube.model.CommentSnippet; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import io.mattw.youtube.commentsuite.util.UTF8UrlEncoder; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; - -import java.io.IOException; -import java.lang.reflect.Modifier; - -/** - * @author mattwright324 - */ -public class OAuth2Handler { - - private static final Logger logger = LogManager.getLogger(); - - private Gson gson = new GsonBuilder().excludeFieldsWithModifiers(Modifier.FINAL).create(); - private String clientId; - private String clientSecret; - private String redirectUri; - private String authUrl; - private OAuth2Tokens tokens; - - OAuth2Handler(String clientId, String clientSecret, String redirectUri) { - this.clientId = clientId; - this.clientSecret = clientSecret; - this.redirectUri = redirectUri; - - this.authUrl = String.format("https://accounts.google.com/o/oauth2/auth?client_id=%s&redirect_uri=%s&&response_type=code&&scope=%s", - clientId, - UTF8UrlEncoder.encode(redirectUri), - UTF8UrlEncoder.encode("https://www.googleapis.com/auth/youtube.force-ssl")); - } - - public OAuth2Tokens getTokens() { - return tokens; - } - - public void setTokens(OAuth2Tokens tokens) { - this.tokens = tokens; - } - - public String getAuthUrl() { - return authUrl; - } - - public OAuth2Tokens getAccessTokens(String code) throws IOException { - Document doc = Jsoup.connect("https://accounts.google.com/o/oauth2/token") - .ignoreContentType(true) - .data("code", code) - .data("client_id", clientId) - .data("client_secret", clientSecret) - .data("redirect_uri", redirectUri) - .data("grant_type", "authorization_code") - .post(); - return gson.fromJson(doc.text(), OAuth2Tokens.class); - } - - public void refreshTokens() throws IOException { - Document doc = Jsoup.connect("https://accounts.google.com/o/oauth2/token") - .ignoreContentType(true) - .data("client_id", clientId) - .data("client_secret", clientSecret) - .data("refresh_token", tokens.getRefreshToken()) - .data("grant_type", "refresh_token") - .post(); - OAuth2Tokens newTokens = gson.fromJson(doc.text(), OAuth2Tokens.class); - newTokens.setRefreshToken(tokens.getRefreshToken()); - setTokens(newTokens); - } - - /** - * Attempts to send a reply to the parent comment id and text supplied. It will attempt to send to reply 5 times - * after failure and throwing an error. On each failure, if it detects the tokens used by the account have - * expired, it will attempt to refresh them and use and newly updated tokens. - * - * @param parentId id of comment or parentId of reply-comment to reply to - * @param textOriginal text to reply to the comment with - * @throws IOException failed to reply - */ - public Comment postReply(String parentId, String textOriginal) throws IOException { - CommentSnippet snippet = new CommentSnippet(); - snippet.setParentId(parentId); - snippet.setTextOriginal(textOriginal); - - Comment comment = new Comment(); - comment.setSnippet(snippet); - - int attempt = 0; - do { - try { - Comment result = FXMLSuite.getYouTube().comments() - .insert("snippet", comment) - .setOauthToken(tokens.getAccessToken()) - .execute(); - - logger.debug("Successfully replied [id={}]", result.getId()); - - return result; - } catch (IOException e) { - logger.warn("Failed on comment reply, {}", e.getLocalizedMessage()); - logger.debug("Refreshing tokens and trying again [attempt={}]", attempt); - - refreshTokens(); - } - - attempt++; - } while (attempt < 5); - - throw new IOException("Could not reply and failed to refresh tokens."); - } -} diff --git a/src/main/java/io/mattw/youtube/commentsuite/OAuth2Tokens.java b/src/main/java/io/mattw/youtube/commentsuite/OAuth2Tokens.java deleted file mode 100644 index 78b7693..0000000 --- a/src/main/java/io/mattw/youtube/commentsuite/OAuth2Tokens.java +++ /dev/null @@ -1,32 +0,0 @@ -package io.mattw.youtube.commentsuite; - -import java.io.Serializable; - -/** - * @author mattwright324 - */ -public class OAuth2Tokens implements Serializable { - - private String access_token, token_type, refresh_token; - private int expires_in; - - public String getAccessToken() { - return access_token; - } - - public String getTokenType() { - return token_type; - } - - public String getRefreshToken() { - return refresh_token; - } - - public void setRefreshToken(String token) { - refresh_token = token; - } - - public int getExpiresIn() { - return expires_in; - } -} diff --git a/src/main/java/io/mattw/youtube/commentsuite/YouTubeAccount.java b/src/main/java/io/mattw/youtube/commentsuite/YouTubeAccount.java deleted file mode 100644 index 0edff1b..0000000 --- a/src/main/java/io/mattw/youtube/commentsuite/YouTubeAccount.java +++ /dev/null @@ -1,119 +0,0 @@ -package io.mattw.youtube.commentsuite; - -import com.google.api.client.googleapis.json.GoogleJsonResponseException; -import com.google.api.services.youtube.YouTube; -import com.google.api.services.youtube.model.Channel; -import com.google.api.services.youtube.model.ChannelListResponse; -import io.mattw.youtube.commentsuite.db.CommentDatabase; -import io.mattw.youtube.commentsuite.db.YouTubeChannel; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.io.IOException; -import java.io.Serializable; -import java.sql.SQLException; -import java.util.Collections; - -/** - * Combination of YouTubeChannel and OAuth2Tokens as sign-in. - * Data stored in Config 'commentsuite.json' - * - * TODO: Update tokens on app start each time? - * Currently only refreshing tokens when they are most needed, when posting a reply. - * - {@link OAuth2Handler#postReply(String, String)} - * - * @author mattwright324 - */ -public class YouTubeAccount implements Serializable { - - private static final transient Logger logger = LogManager.getLogger(); - - private String username, channelId, thumbUrl; - private OAuth2Tokens tokens; - - public YouTubeAccount() { - // default constructor - } - - /** - * Only used with "YouTube Account Sign-in," otherwise initialized by Gson & Config. - */ - public YouTubeAccount(OAuth2Tokens tokens) { - this.tokens = tokens; - - updateData(); - } - - /** - * Using the OAuth2Tokens passed in, query the YouTube API to get the "mine" - * channel for those tokens. - *

- * Pushes the channel to the database. - */ - void updateData() { - CommentDatabase database = FXMLSuite.getDatabase(); - YouTube youtube = FXMLSuite.getYouTube(); - - logger.debug("Getting account data for [username={}]", getUsername()); - - try { - ChannelListResponse cl = youtube.channels().list("snippet") - .setOauthToken(getTokens().getAccessToken()) - .setMine(true) - .execute(); - - Channel cli = cl.getItems().get(0); - - this.channelId = cli.getId(); - - if (cli.getSnippet() != null) { - this.username = cli.getSnippet().getTitle(); - this.thumbUrl = cli.getSnippet().getThumbnails().getMedium().getUrl(); - - try { - YouTubeChannel channel = new YouTubeChannel(cli); - database.insertChannels(Collections.singletonList(channel)); - database.commit(); - } catch (SQLException e) { - logger.error("Unable to insert account channel into database.", e); - } - } - } catch (GoogleJsonResponseException e) { - if (e.getStatusCode() == 401) { - logger.warn("Tokens have expired for account [username={}]", getUsername()); - } else { - logger.error("An unexpected error occurred.", e); - } - } catch (IOException e) { - logger.error("Failed to query for account channel info.", e); - } - } - - public String getUsername() { - return username; - } - - public String getChannelId() { - return channelId; - } - - public String getThumbUrl() { - return thumbUrl; - } - - public String toString() { - return username; - } - - public OAuth2Tokens getTokens() { - return tokens; - } - - public void setTokens(OAuth2Tokens tokens) { - this.tokens = tokens; - } - - public boolean equals(Object o) { - return o instanceof YouTubeAccount && ((YouTubeAccount) o).getChannelId() != null && ((YouTubeAccount) o).getChannelId().equals(channelId); - } -} diff --git a/src/main/java/io/mattw/youtube/commentsuite/db/CommentDatabase.java b/src/main/java/io/mattw/youtube/commentsuite/db/CommentDatabase.java index f3cb4fe..a23b06c 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/db/CommentDatabase.java +++ b/src/main/java/io/mattw/youtube/commentsuite/db/CommentDatabase.java @@ -3,7 +3,7 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.google.common.eventbus.EventBus; -import io.mattw.youtube.commentsuite.FXMLSuite; +import io.mattw.youtube.commentsuite.CommentSuite; import io.mattw.youtube.commentsuite.events.*; import javafx.collections.FXCollections; import javafx.collections.ObservableList; @@ -17,9 +17,8 @@ import java.util.*; import java.util.concurrent.TimeUnit; -/** - * @author mattwright324 - */ +import static io.mattw.youtube.commentsuite.db.SQLLoader.*; + public class CommentDatabase implements Closeable { private static final Logger logger = LogManager.getLogger(); @@ -32,7 +31,7 @@ public class CommentDatabase implements Closeable { private final Cache channelCache = CacheBuilder.newBuilder() .expireAfterAccess(5, TimeUnit.MINUTES) .build(); - private final EventBus eventBus = FXMLSuite.getEventBus(); + private final EventBus eventBus = CommentSuite.getEventBus(); /** * Default constructor for testing. @@ -80,7 +79,7 @@ public void commit() throws SQLException { protected void create() throws SQLException { logger.debug("Creating tables if not exists."); try (Statement s = sqlite.createStatement()) { - s.executeUpdate(SQLLoader.CREATE_DB.toString()); + s.executeUpdate(CREATE_DB.toString()); } this.commit(); } @@ -88,7 +87,7 @@ protected void create() throws SQLException { public void reset() throws SQLException { logger.warn("Dropping all database contents. This cannot be undone."); try (Statement s = sqlite.createStatement()) { - s.executeUpdate(SQLLoader.RESET_DB.toString()); + s.executeUpdate(RESET_DB.toString()); } this.commit(); this.vacuum(); @@ -99,7 +98,7 @@ public void vacuum() throws SQLException { logger.warn("Vacuuming database. This may take a long time."); sqlite.setAutoCommit(true); try (Statement s = sqlite.createStatement()) { - s.execute(SQLLoader.VACUUM_DB.toString()); + s.execute(VACUUM_DB.toString()); } sqlite.setAutoCommit(false); } @@ -107,7 +106,7 @@ public void vacuum() throws SQLException { public void cleanUp() throws SQLException { logger.warn("Cleaning database of unlinked content."); try (Statement s = sqlite.createStatement()) { - s.executeUpdate(SQLLoader.CLEAN_DB.toString()); + s.executeUpdate(CLEAN_DB.toString()); } commit(); } @@ -126,7 +125,7 @@ public List getAllGroups() { private void refreshAllGroups() { allGroups.clear(); try (final Statement s = sqlite.createStatement(); - final ResultSet rs = s.executeQuery(SQLLoader.GET_ALL_GROUPS.toString())) { + final ResultSet rs = s.executeQuery(GET_ALL_GROUPS.toString())) { while (rs.next()) { final Group group = resultSetToGroup(rs); @@ -175,7 +174,8 @@ public YouTubeComment resultSetToComment(ResultSet rs) throws SQLException { rs.getInt("comment_likes"), rs.getInt("reply_count"), rs.getBoolean("is_reply"), - rs.getString("parent_id")); + rs.getString("parent_id"), + hasColumn(rs, "moderation_status") ? rs.getString("moderation_status") : null); } private YouTubeVideo resultSetToVideo(ResultSet rs) throws SQLException { @@ -193,8 +193,19 @@ private YouTubeVideo resultSetToVideo(ResultSet rs) throws SQLException { rs.getInt("http_code")); } + public static boolean hasColumn(ResultSet rs, String columnName) throws SQLException { + ResultSetMetaData rsmd = rs.getMetaData(); + int columns = rsmd.getColumnCount(); + for (int x = 1; x <= columns; x++) { + if (columnName.equals(rsmd.getColumnName(x))) { + return true; + } + } + return false; + } + public boolean doesChannelNotExist(String channelId) { - try (PreparedStatement ps = sqlite.prepareStatement(SQLLoader.DOES_CHANNEL_EXIST.toString())) { + try (PreparedStatement ps = sqlite.prepareStatement(DOES_CHANNEL_EXIST.toString())) { ps.setString(1, channelId); try (ResultSet rs = ps.executeQuery()) { return !rs.next(); @@ -228,7 +239,7 @@ public Collection findChannelsNotExisting(Collection channelIds) } public boolean doesVideoExist(String videoId) { - try (PreparedStatement ps = sqlite.prepareStatement(SQLLoader.DOES_VIDEO_EXIST.toString())) { + try (PreparedStatement ps = sqlite.prepareStatement(DOES_VIDEO_EXIST.toString())) { ps.setString(1, videoId); try (ResultSet rs = ps.executeQuery()) { return rs.next(); @@ -251,7 +262,20 @@ public long countVideosNotExisting(Collection videoIds) { } public boolean doesCommentExist(String commentId) { - try (PreparedStatement ps = sqlite.prepareStatement(SQLLoader.DOES_COMMENT_EXIST.toString())) { + try (PreparedStatement ps = sqlite.prepareStatement(DOES_COMMENT_EXIST.toString())) { + ps.setString(1, commentId); + try (ResultSet rs = ps.executeQuery()) { + return rs.next(); + } + } catch (SQLException e) { + logger.error(e); + e.printStackTrace(); + } + return false; + } + + public boolean doesModeratedCommentExist(String commentId) { + try (PreparedStatement ps = sqlite.prepareStatement(DOES_MODERATED_COMMENT_EXIST.toString())) { ps.setString(1, commentId); try (ResultSet rs = ps.executeQuery()) { return rs.next(); @@ -274,13 +298,24 @@ public long countCommentsNotExisting(Collection commentIds) { return notExists; } + public long countModeratedCommentsNotExisting(Collection commentIds) { + // logger.trace("Checking if commentIds's exist [size={},ids={}]", commentIds.size(), commentIds.toString()); + long notExists = 0; + for (String id : commentIds) { + if (!doesModeratedCommentExist(id)) { + notExists++; + } + } + return notExists; + } + /** * Returns list of GroupItems for a given Group. * If no GroupItems are present, returns a "No groups" Item with id GroupItem.NO_ITEMS */ public List getGroupItems(Group g) { List items = new ArrayList<>(); - try (PreparedStatement ps = sqlite.prepareStatement(SQLLoader.GET_GROUPITEMS.toString())) { + try (PreparedStatement ps = sqlite.prepareStatement(GET_GROUPITEMS.toString())) { ps.setString(1, g.getGroupId()); try (ResultSet rs = ps.executeQuery()) { @@ -318,7 +353,7 @@ public Group createGroup(String name) throws SQLException { public Group createGroup(Group group) throws SQLException { logger.trace("Created Group [name={}, id={}]", group.getName(), group.getGroupId()); - try (PreparedStatement ps = sqlite.prepareStatement(SQLLoader.GROUP_CREATE.toString())) { + try (PreparedStatement ps = sqlite.prepareStatement(GROUP_CREATE.toString())) { ps.setString(1, group.getGroupId()); ps.setString(2, group.getName()); ps.executeUpdate(); @@ -340,7 +375,7 @@ public void renameGroup(Group group, String newName) throws SQLException { if (!group.getName().equals(newName)) { logger.trace("Renaming Group [id={},name={},newName={}]", group.getGroupId(), group.getName(), newName); - try (PreparedStatement ps = sqlite.prepareStatement(SQLLoader.GROUP_RENAME.toString())) { + try (PreparedStatement ps = sqlite.prepareStatement(GROUP_RENAME.toString())) { ps.setString(1, newName); ps.setString(2, group.getGroupId()); ps.executeUpdate(); @@ -358,7 +393,7 @@ public void renameGroup(Group group, String newName) throws SQLException { * Recommended to run cleanUp() afterwards. */ public void deleteGroup(Group group) throws SQLException { - try (PreparedStatement ps = sqlite.prepareStatement(SQLLoader.DELETE_GROUP.toString())) { + try (PreparedStatement ps = sqlite.prepareStatement(DELETE_GROUP.toString())) { ps.setString(1, group.getGroupId()); ps.executeUpdate(); @@ -376,8 +411,8 @@ public void deleteGroup(Group group) throws SQLException { * Could create links to the same GroupItem to multiple Group(s). */ public void insertGroupItems(Group group, List items) throws SQLException { - try (PreparedStatement psCG = sqlite.prepareStatement(SQLLoader.CREATE_GITEM.toString()); - PreparedStatement psCGG = sqlite.prepareStatement(SQLLoader.CREATE_GROUP_GITEM.toString())) { + try (PreparedStatement psCG = sqlite.prepareStatement(CREATE_GITEM.toString()); + PreparedStatement psCGG = sqlite.prepareStatement(CREATE_GROUP_GITEM.toString())) { for (GroupItem gi : items) { psCG.setString(1, gi.getId()); @@ -404,7 +439,7 @@ public void insertGroupItems(Group group, List items) throws SQLExcep * Updates gitem_list with set lastChecked values. */ public void updateGroupItem(GroupItem item) throws SQLException { - try (PreparedStatement ps = sqlite.prepareStatement(SQLLoader.UPDATE_GITEM.toString())) { + try (PreparedStatement ps = sqlite.prepareStatement(UPDATE_GITEM.toString())) { ps.setString(1, item.getTitle()); ps.setString(2, item.getChannelTitle()); ps.setLong(3, item.getPublished()); @@ -420,7 +455,7 @@ public void updateGroupItem(GroupItem item) throws SQLException { * Recommended to run cleanUp() afterwards. */ public void deleteGroupItemLinks(Group group, List items) throws SQLException { - try (PreparedStatement ps = sqlite.prepareStatement(SQLLoader.DELETE_GROUP_GITEM.toString())) { + try (PreparedStatement ps = sqlite.prepareStatement(DELETE_GROUP_GITEM.toString())) { for (GroupItem gi : items) { ps.setString(1, gi.getId()); ps.setString(2, group.getGroupId()); @@ -437,7 +472,7 @@ public void deleteGroupItemLinks(Group group, List items) throws SQLE */ public void insertComments(List items) throws SQLException { // logger.trace("Inserting Comments [size={}]", items.size()); - try (PreparedStatement ps = sqlite.prepareStatement(SQLLoader.INSERT_IGNORE_COMMENTS.toString())) { + try (PreparedStatement ps = sqlite.prepareStatement(INSERT_IGNORE_COMMENTS.toString())) { for (YouTubeComment ct : items) { ps.setString(1, ct.getId()); ps.setString(2, ct.getChannelId()); @@ -452,6 +487,47 @@ public void insertComments(List items) throws SQLException { } ps.executeBatch(); } + + // If we got the same comment this time from the published, delete it in moderated. + try (PreparedStatement ps = sqlite.prepareStatement(DELETE_MODERATED_COMMENT.toString())) { + for (YouTubeComment comment : items) { + ps.setString(1, comment.getId()); + ps.addBatch(); + } + ps.executeBatch(); + } + } + + /** + * Inserts moderatedcomments for group refreshing. + */ + public void insertModeratedComments(List items) throws SQLException { + // logger.trace("Inserting Comments [size={}]", items.size()); + try (PreparedStatement ps = sqlite.prepareStatement(INSERT_IGNORE_MODERATED_COMMENTS.toString())) { + for (YouTubeComment ct : items) { + ps.setString(1, ct.getId()); + ps.setString(2, ct.getChannelId()); + ps.setString(3, ct.getVideoId()); + ps.setLong(4, ct.getPublished()); + ps.setString(5, ct.getCommentText()); + ps.setLong(6, ct.getLikes()); + ps.setLong(7, ct.getReplyCount()); + ps.setBoolean(8, ct.isReply()); + ps.setString(9, ct.getParentId()); + ps.setString(10, ct.getModerationStatus().name()); + ps.addBatch(); + } + ps.executeBatch(); + } + + // If a comment was added from in-app reply, remove from the published-comments table since it is actually moderated. + try (PreparedStatement ps = sqlite.prepareStatement(DELETE_COMMENT.toString())) { + for (YouTubeComment comment : items) { + ps.setString(1, comment.getId()); + ps.addBatch(); + } + ps.executeBatch(); + } } /** @@ -459,7 +535,7 @@ public void insertComments(List items) throws SQLException { * Used for group refresh. */ public List getCommentIds(Group group) throws SQLException { - try (PreparedStatement ps = sqlite.prepareStatement(SQLLoader.GET_ALL_COMMENT_IDS_BY_GROUP.toString())) { + try (PreparedStatement ps = sqlite.prepareStatement(GET_ALL_COMMENT_IDS_BY_GROUP.toString())) { ps.setString(1, group.getGroupId()); try (ResultSet rs = ps.executeQuery()) { List list = new ArrayList<>(); @@ -478,7 +554,7 @@ public List getCommentIds(Group group) throws SQLException { */ public void insertVideos(List items) throws SQLException { logger.debug("Inserting Videos [size={}]", items.size()); - try (PreparedStatement ps = sqlite.prepareStatement(SQLLoader.INSERT_REPLACE_VIDEOS.toString())) { + try (PreparedStatement ps = sqlite.prepareStatement(INSERT_REPLACE_VIDEOS.toString())) { for (YouTubeVideo video : items) { ps.setString(1, video.getId()); ps.setString(2, video.getChannelId()); @@ -504,7 +580,7 @@ public void insertVideos(List items) throws SQLException { */ public List getAllVideoIds() throws SQLException { try (Statement s = sqlite.createStatement(); - ResultSet rs = s.executeQuery(SQLLoader.GET_ALL_UNIQUE_VIDEO_IDS.toString())) { + ResultSet rs = s.executeQuery(GET_ALL_UNIQUE_VIDEO_IDS.toString())) { List ids = new ArrayList<>(); while (rs.next()) { @@ -521,7 +597,7 @@ public List getAllVideoIds() throws SQLException { */ public List getVideoIds(Group group) throws SQLException { logger.debug("GET_ALL_VIDEO_IDS_BY_GROUP"); - try (PreparedStatement ps = sqlite.prepareStatement(SQLLoader.GET_ALL_VIDEO_IDS_BY_GROUP.toString())) { + try (PreparedStatement ps = sqlite.prepareStatement(GET_ALL_VIDEO_IDS_BY_GROUP.toString())) { ps.setString(1, group.getGroupId()); try (ResultSet rs = ps.executeQuery()) { List list = new ArrayList<>(); @@ -541,7 +617,7 @@ public List getVideoIds(Group group) throws SQLException { */ public List getVideoIds(GroupItem gitem) throws SQLException { logger.debug("GET_ALL_VIDEO_IDS_BY_GROUPITEM"); - try (PreparedStatement ps = sqlite.prepareStatement(SQLLoader.GET_ALL_VIDEO_IDS_BY_GROUPITEM.toString())) { + try (PreparedStatement ps = sqlite.prepareStatement(GET_ALL_VIDEO_IDS_BY_GROUPITEM.toString())) { ps.setString(1, gitem.getId()); try (ResultSet rs = ps.executeQuery()) { List list = new ArrayList<>(); @@ -556,7 +632,7 @@ public List getVideoIds(GroupItem gitem) throws SQLException { public YouTubeVideo getVideo(String videoId) throws SQLException { - try (PreparedStatement ps = sqlite.prepareStatement(SQLLoader.GET_VIDEO_BY_ID.toString())) { + try (PreparedStatement ps = sqlite.prepareStatement(GET_VIDEO_BY_ID.toString())) { ps.setString(1, videoId); try (ResultSet rs = ps.executeQuery()) { if (rs.next()) { @@ -569,7 +645,7 @@ public YouTubeVideo getVideo(String videoId) throws SQLException { } public List getVideos(GroupItem gitem, String keyword, String order, int limit) throws SQLException { - try (PreparedStatement ps = sqlite.prepareStatement(SQLLoader.GET_VIDEOS_BY_CRITERIA_GITEM.toString() + try (PreparedStatement ps = sqlite.prepareStatement(GET_VIDEOS_BY_CRITERIA_GITEM.toString() .replace(":order", order))) { ps.setString(1, gitem.getId()); ps.setString(2, "%" + keyword + "%"); @@ -581,7 +657,7 @@ public List getVideos(GroupItem gitem, String keyword, String orde } public List getVideos(Group group, String keyword, String order, int limit) throws SQLException { - try (PreparedStatement ps = sqlite.prepareStatement(SQLLoader.GET_VIDEOS_BY_CRITERIA_GROUP.toString() + try (PreparedStatement ps = sqlite.prepareStatement(GET_VIDEOS_BY_CRITERIA_GROUP.toString() .replace(":order", order))) { ps.setString(1, group.getGroupId()); ps.setString(2, "%" + keyword + "%"); @@ -617,7 +693,7 @@ public List getVideos(Group group, String keyword, String order, i * Updates http code for group refreshing. */ public void updateVideoHttpCode(String videoId, int httpCode) throws SQLException { - try (PreparedStatement ps = sqlite.prepareStatement(SQLLoader.UPDATE_VIDEO_HTTPCODE.toString())) { + try (PreparedStatement ps = sqlite.prepareStatement(UPDATE_VIDEO_HTTPCODE.toString())) { ps.setInt(1, httpCode); ps.setString(2, videoId); ps.executeUpdate(); @@ -647,7 +723,7 @@ public void updateVideoHttpCode(String videoId, int httpCode) throws SQLExceptio */ public void insertChannels(List items) throws SQLException { // logger.debug("Inserting Channels [size={}]", items.size()); - try (PreparedStatement ps = sqlite.prepareStatement(SQLLoader.INSERT_IGNORE_CHANNELS.toString())) { + try (PreparedStatement ps = sqlite.prepareStatement(INSERT_IGNORE_CHANNELS.toString())) { for (YouTubeChannel c : items) { ps.setString(1, c.getId()); ps.setString(2, c.getTitle()); @@ -679,7 +755,7 @@ public void insertChannels(List items) throws SQLException { * Checks if exists and caches it if it does. */ public YouTubeChannel channelExists(String channelId) throws SQLException { - try (PreparedStatement ps = sqlite.prepareStatement(SQLLoader.GET_CHANNEL_EXISTS.toString())) { + try (PreparedStatement ps = sqlite.prepareStatement(GET_CHANNEL_EXISTS.toString())) { ps.setString(1, channelId); try (ResultSet rs = ps.executeQuery()) { YouTubeChannel channel = null; @@ -732,7 +808,7 @@ public void insertGroupItemVideo(GroupItemVideo item) throws SQLException { * Inserts to gitem_video for group refreshing. */ public void insertGroupItemVideo(List items) throws SQLException { - try (PreparedStatement ps = sqlite.prepareStatement(SQLLoader.INSERT_IGNORE_GITEM_VIDEO.toString())) { + try (PreparedStatement ps = sqlite.prepareStatement(INSERT_IGNORE_GITEM_VIDEO.toString())) { for (GroupItemVideo vg : items) { ps.setString(1, vg.getGitemId()); ps.setString(2, vg.getVideoId()); @@ -764,10 +840,13 @@ public CommentQuery commentQuery() { /** * Returns all of the comments associated with a comment parentId. */ - public List getCommentTree(String parentId) throws SQLException { - try (PreparedStatement ps = sqlite.prepareStatement(SQLLoader.GET_COMMENT_TREE.toString())) { + public List getCommentTree(String parentId, boolean includeModeration) throws SQLException { + try (PreparedStatement ps = sqlite.prepareStatement(GET_COMMENT_TREE.toString())) { ps.setString(1, parentId); ps.setString(2, parentId); + ps.setBoolean(3, includeModeration); + ps.setString(4, parentId); + ps.setString(5, parentId); try (ResultSet rs = ps.executeQuery()) { List tree = new ArrayList<>(); while (rs.next()) { @@ -782,7 +861,7 @@ public List getCommentTree(String parentId) throws SQLException /******** Stats and Info Methods **********/ public long getLastChecked(Group group) { - try (PreparedStatement ps = sqlite.prepareStatement(SQLLoader.GET_GROUP_LAST_CHECKED.toString())) { + try (PreparedStatement ps = sqlite.prepareStatement(GET_GROUP_LAST_CHECKED.toString())) { ps.setString(1, group.getGroupId()); try (ResultSet rs = ps.executeQuery()) { if (rs.next()) { @@ -797,7 +876,7 @@ public long getLastChecked(Group group) { public GroupStats getGroupStats(Group group) throws SQLException { GroupStats stats = new GroupStats(); - try (PreparedStatement ps = sqlite.prepareStatement(SQLLoader.GET_VIDEO_STATS.toString())) { + try (PreparedStatement ps = sqlite.prepareStatement(GET_VIDEO_STATS.toString())) { ps.setString(1, group.getGroupId()); try (ResultSet rs = ps.executeQuery()) { if (rs.next()) { @@ -809,7 +888,7 @@ public GroupStats getGroupStats(Group group) throws SQLException { } } } - try (PreparedStatement ps = sqlite.prepareStatement(SQLLoader.GET_UNIQUE_VIEWERS_BY_GROUP.toString())) { + try (PreparedStatement ps = sqlite.prepareStatement(GET_UNIQUE_VIEWERS_BY_GROUP.toString())) { ps.setString(1, group.getGroupId()); try (ResultSet rs = ps.executeQuery()) { if (rs.next()) { @@ -823,7 +902,7 @@ public GroupStats getGroupStats(Group group) throws SQLException { stats.setCommentsDisabled(this.getDisabledVideos(group, 25)); stats.setWeeklyUploadHistogram(this.getWeekByWeekVideoHistogram(group)); - try (PreparedStatement ps = sqlite.prepareStatement(SQLLoader.GET_COMMENT_STATS.toString())) { + try (PreparedStatement ps = sqlite.prepareStatement(GET_COMMENT_STATS.toString())) { ps.setString(1, group.getGroupId()); try (ResultSet rs = ps.executeQuery()) { if (rs.next()) { @@ -832,6 +911,14 @@ public GroupStats getGroupStats(Group group) throws SQLException { } } } + try (PreparedStatement ps = sqlite.prepareStatement(GET_MODERATED_COMMENT_STATS.toString())) { + ps.setString(1, group.getGroupId()); + try (ResultSet rs = ps.executeQuery()) { + if (rs.next()) { + stats.setTotalModeratedComments(rs.getLong("total_comments")); + } + } + } stats.setMostLikedViewers(this.getMostPopularViewers(group, 25)); stats.setMostActiveViewers(this.getMostActiveViewers(group, 25)); stats.setWeeklyCommentHistogram(this.getWeekByWeekCommentHistogram(group)); @@ -840,21 +927,21 @@ public GroupStats getGroupStats(Group group) throws SQLException { } private Map getWeekByWeekCommentHistogram(Group group) throws SQLException { - try (PreparedStatement ps = sqlite.prepareStatement(SQLLoader.GET_COMMENT_WEEK_HISTOGRAM.toString())) { + try (PreparedStatement ps = sqlite.prepareStatement(GET_COMMENT_WEEK_HISTOGRAM.toString())) { ps.setString(1, group.getGroupId()); return resultSetToHistogram(ps); } } private Map getWeekByWeekVideoHistogram(Group group) throws SQLException { - try (PreparedStatement ps = sqlite.prepareStatement(SQLLoader.GET_VIDEO_WEEK_HISTOGRAM.toString())) { + try (PreparedStatement ps = sqlite.prepareStatement(GET_VIDEO_WEEK_HISTOGRAM.toString())) { ps.setString(1, group.getGroupId()); return resultSetToHistogram(ps); } } private LinkedHashMap getMostActiveViewers(Group group, int limit) throws SQLException { - try (PreparedStatement ps = sqlite.prepareStatement(SQLLoader.GET_GROUP_ACTIVE_VIEWERS.toString())) { + try (PreparedStatement ps = sqlite.prepareStatement(GET_GROUP_ACTIVE_VIEWERS.toString())) { ps.setString(1, group.getGroupId()); ps.setInt(2, limit); try (ResultSet rs = ps.executeQuery()) { @@ -869,7 +956,7 @@ private LinkedHashMap getMostActiveViewers(Group group, in } private LinkedHashMap getMostPopularViewers(Group group, int limit) throws SQLException { - try (PreparedStatement ps = sqlite.prepareStatement(SQLLoader.GET_GROUP_POPULAR_VIEWERS.toString())) { + try (PreparedStatement ps = sqlite.prepareStatement(GET_GROUP_POPULAR_VIEWERS.toString())) { ps.setString(1, group.getGroupId()); ps.setInt(2, limit); try (ResultSet rs = ps.executeQuery()) { @@ -884,7 +971,7 @@ private LinkedHashMap getMostPopularViewers(Group group, i } private List getMostPopularVideos(Group group, int limit) throws SQLException { - try (PreparedStatement ps = sqlite.prepareStatement(SQLLoader.GET_GROUP_POPULAR_VIDEOS.toString())) { + try (PreparedStatement ps = sqlite.prepareStatement(GET_GROUP_POPULAR_VIDEOS.toString())) { ps.setString(1, group.getGroupId()); ps.setInt(2, limit); return resultSetToVideoList(ps); @@ -892,7 +979,7 @@ private List getMostPopularVideos(Group group, int limit) throws S } private List getMostDislikedVideos(Group group, int limit) throws SQLException { - try (PreparedStatement ps = sqlite.prepareStatement(SQLLoader.GET_GROUP_DISLIKED_VIDEOS.toString())) { + try (PreparedStatement ps = sqlite.prepareStatement(GET_GROUP_DISLIKED_VIDEOS.toString())) { ps.setString(1, group.getGroupId()); ps.setInt(2, limit); return resultSetToVideoList(ps); @@ -900,7 +987,7 @@ private List getMostDislikedVideos(Group group, int limit) throws } private List getMostCommentedVideos(Group group, int limit) throws SQLException { - try (PreparedStatement ps = sqlite.prepareStatement(SQLLoader.GET_GROUP_COMMENTED_VIDEOS.toString())) { + try (PreparedStatement ps = sqlite.prepareStatement(GET_GROUP_COMMENTED_VIDEOS.toString())) { ps.setString(1, group.getGroupId()); ps.setInt(2, limit); return resultSetToVideoList(ps); @@ -908,7 +995,7 @@ private List getMostCommentedVideos(Group group, int limit) throws } private List getDisabledVideos(Group group, int limit) throws SQLException { - try (PreparedStatement ps = sqlite.prepareStatement(SQLLoader.GET_GROUP_DISABLED_VIDEOS.toString())) { + try (PreparedStatement ps = sqlite.prepareStatement(GET_GROUP_DISABLED_VIDEOS.toString())) { ps.setString(1, group.getGroupId()); ps.setInt(2, limit); return resultSetToVideoList(ps); diff --git a/src/main/java/io/mattw/youtube/commentsuite/db/CommentQuery.java b/src/main/java/io/mattw/youtube/commentsuite/db/CommentQuery.java index 10724cf..e9cd80e 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/db/CommentQuery.java +++ b/src/main/java/io/mattw/youtube/commentsuite/db/CommentQuery.java @@ -14,10 +14,11 @@ import java.util.*; import java.util.stream.Collectors; +import static io.mattw.youtube.commentsuite.db.CommentQuery.CommentsType.*; + /** * Queries the database for comments. * - * @author mattwright324 */ public class CommentQuery implements Serializable, Exportable { @@ -87,10 +88,12 @@ private String buildQueryStringAndParams() { } } - List queryLines = new ArrayList<>(); - queryLines.add("SELECT * FROM comments"); + final String commentsTable = commentsType == MODERATED_ONLY ? "comments_moderated" : "comments"; + final List queryLines = new ArrayList<>(); + queryLines.add("SELECT * FROM"); + queryLines.add(commentsTable); queryLines.add("LEFT JOIN channels USING (channel_id)"); - queryLines.add("WHERE comments.video_id IN (:videoSubquery)".replace(":videoSubquery", videoSubquery)); + queryLines.add("WHERE " + commentsTable + ".video_id IN (:videoSubquery)".replace(":videoSubquery", videoSubquery)); if (StringUtils.isNotEmpty(nameLike)) { queryLines.add("AND (channels.channel_name LIKE :nameLike OR channels.channel_id = :channelId)"); @@ -98,18 +101,18 @@ private String buildQueryStringAndParams() { queryParams.put("channelId", nameLike); } if (StringUtils.isNotEmpty(textLike)) { - queryLines.add("AND (comments.comment_text LIKE :textLike OR comments.comment_id = :commentId)"); + queryLines.add("AND (" + commentsTable + ".comment_text LIKE :textLike OR " + commentsTable + ".comment_id = :commentId)"); queryParams.put("textLike", '%' + textLike + '%'); queryParams.put("commentId", textLike); } - if (commentsType != CommentsType.ALL) { + if (commentsType == REPLIES_ONLY || commentsType == COMMENTS_ONLY) { queryLines.add("AND is_reply = :isReply"); - queryParams.put("isReply", commentsType == CommentsType.REPLIES_ONLY); + queryParams.put("isReply", commentsType == REPLIES_ONLY); } - queryLines.add("AND comments.comment_date > :dateFrom AND comments.comment_date < :dateTo"); + queryLines.add("AND " + commentsTable + ".comment_date > :dateFrom AND " + commentsTable + ".comment_date < :dateTo"); queryParams.put("dateFrom", localDateToEpochMillis(dateFrom, false)); queryParams.put("dateTo", localDateToEpochMillis(dateTo, true)); @@ -394,6 +397,7 @@ public enum CommentsType { ALL("Comments and Replies"), COMMENTS_ONLY("Comments Only"), REPLIES_ONLY("Replies Only"), + MODERATED_ONLY("Moderated Only") ; private String title; diff --git a/src/main/java/io/mattw/youtube/commentsuite/db/Group.java b/src/main/java/io/mattw/youtube/commentsuite/db/Group.java index 40b327f..a05bd1f 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/db/Group.java +++ b/src/main/java/io/mattw/youtube/commentsuite/db/Group.java @@ -4,9 +4,6 @@ import java.nio.charset.StandardCharsets; import java.security.MessageDigest; -/** - * @author mattwright324 - */ public class Group { private final String groupId; diff --git a/src/main/java/io/mattw/youtube/commentsuite/db/GroupItem.java b/src/main/java/io/mattw/youtube/commentsuite/db/GroupItem.java index 7e0d37b..38c55f3 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/db/GroupItem.java +++ b/src/main/java/io/mattw/youtube/commentsuite/db/GroupItem.java @@ -3,7 +3,7 @@ import com.google.api.services.youtube.YouTube; import com.google.api.services.youtube.model.*; import io.mattw.youtube.commentsuite.ConfigData; -import io.mattw.youtube.commentsuite.FXMLSuite; +import io.mattw.youtube.commentsuite.CommentSuite; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -17,8 +17,6 @@ /** * Database entry for searched YouTube entries (Video, Channel, Playlist). * getYouTubeId() from YouTubeObject represents the GroupItem ID. - * - * @author mattwright324 */ public class GroupItem extends YouTubeObject { @@ -37,7 +35,7 @@ public class GroupItem extends YouTubeObject { /** * Used for converting selected search items for inserting into database. */ - public GroupItem(SearchResult item) { + public GroupItem(final SearchResult item) { super(item.getId(), item.getSnippet().getTitle(), item.getSnippet().getThumbnails().getMedium().getUrl()); @@ -55,7 +53,7 @@ public GroupItem(SearchResult item) { * * @param item VideosList.Item */ - public GroupItem(Video item) { + public GroupItem(final Video item) { this(item.getId(), YType.VIDEO, item.getSnippet().getTitle(), item.getSnippet().getChannelTitle(), item.getSnippet().getThumbnails().getMedium().getUrl(), item.getSnippet().getPublishedAt().getValue(), 0); } @@ -65,7 +63,7 @@ public GroupItem(Video item) { * * @param item ChannelsList.Item */ - public GroupItem(Channel item) { + public GroupItem(final Channel item) { this(item.getId(), YType.CHANNEL, item.getSnippet().getTitle(), item.getSnippet().getTitle(), item.getSnippet().getThumbnails().getMedium().getUrl(), item.getSnippet().getPublishedAt().getValue(), 0); } @@ -75,7 +73,7 @@ public GroupItem(Channel item) { * * @param item PlaylistItemsList.Item */ - public GroupItem(Playlist item) { + public GroupItem(final Playlist item) { this(item.getId(), YType.PLAYLIST, item.getSnippet().getTitle(), item.getSnippet().getChannelTitle(), item.getSnippet().getThumbnails().getMedium().getUrl(), item.getSnippet().getPublishedAt().getValue(), 0); } @@ -83,14 +81,14 @@ public GroupItem(Playlist item) { /** * Used for "All Items (#)" and "No items" display in the "Comment Search" and "Group Manager" ComboBoxes. */ - public GroupItem(String gitemId, String title) { + public GroupItem(final String gitemId, final String title) { super(gitemId, title, null); } /** * Used by the database when querying for group items. */ - public GroupItem(String gitemId, YType typeId, String title, String channelTitle, String thumbUrl, long published, long lastChecked) { + public GroupItem(final String gitemId, final YType typeId, final String title, final String channelTitle, final String thumbUrl, final long published, final long lastChecked) { super(gitemId, title, thumbUrl); setTypeId(typeId); this.channelTitle = channelTitle; @@ -104,26 +102,26 @@ public GroupItem(String gitemId, YType typeId, String title, String channelTitle * @param link a video, playlist, or channel link * @throws IOException there was a problem parsing the link */ - public GroupItem(String link) throws IOException { + public GroupItem(final String link) throws IOException { ofLink(link); } - public GroupItem(String link, boolean fastAdd) throws IOException { + public GroupItem(final String link, final boolean fastAdd) throws IOException { this.fastAdd = fastAdd; ofLink(link); } - private void ofLink(String fullLink) throws IOException { + private void ofLink(final String fullLink) throws IOException { logger.debug("Matching link to type [fullLink={}]", fullLink); - youtube = FXMLSuite.getYouTube(); - database = FXMLSuite.getDatabase(); + youtube = CommentSuite.getYouTube(); + database = CommentSuite.getDatabase(); - Pattern video1 = Pattern.compile("(?:http[s]?://youtu.be/)([\\w_\\-]+)"); - Pattern video2 = Pattern.compile("(?:http[s]?://www.youtube.com/watch\\?v=)([\\w_\\-]+)"); - Pattern playlist = Pattern.compile("(?:http[s]?://www.youtube.com/playlist\\?list=)([\\w_\\-]+)"); - Pattern channel1 = Pattern.compile("(?:http[s]?://www.youtube.com/channel/)([\\w_\\-]+)"); - Pattern channel2 = Pattern.compile("(?:http[s]?://www.youtube.com/user/)([\\w_\\-]+)"); + final Pattern video1 = Pattern.compile("(?:http[s]?://youtu.be/)([\\w_\\-]+)"); + final Pattern video2 = Pattern.compile("(?:http[s]?://www.youtube.com/watch\\?v=)([\\w_\\-]+)"); + final Pattern playlist = Pattern.compile("(?:http[s]?://www.youtube.com/playlist\\?list=)([\\w_\\-]+)"); + final Pattern channel1 = Pattern.compile("(?:http[s]?://www.youtube.com/channel/)([\\w_\\-]+)"); + final Pattern channel2 = Pattern.compile("(?:http[s]?://www.youtube.com/user/)([\\w_\\-]+)"); Matcher m; YType type = YType.UNKNOWN; @@ -164,18 +162,18 @@ private void ofLink(String fullLink) throws IOException { } - YouTube youtube = FXMLSuite.getYouTube(); + final YouTube youtube = CommentSuite.getYouTube(); if (result.isEmpty()) { throw new IOException(String.format("Input did not match expected formats [fullLink=%s]", fullLink)); } else { if (type == YType.VIDEO) { - VideoListResponse vl = youtube.videos().list("snippet") - .setKey(FXMLSuite.getYouTubeApiKey()) + final VideoListResponse vl = youtube.videos().list("snippet") + .setKey(CommentSuite.getYouTubeApiKey()) .setId(result) .execute(); if (!vl.getItems().isEmpty()) { - Video item = vl.getItems().get(0); + final Video item = vl.getItems().get(0); duplicate(new GroupItem(item)); @@ -183,7 +181,7 @@ private void ofLink(String fullLink) throws IOException { } } else if (type == YType.CHANNEL) { YouTube.Channels.List cl = youtube.channels().list("snippet") - .setKey(FXMLSuite.getYouTubeApiKey()); + .setKey(CommentSuite.getYouTubeApiKey()); if (!channelUsername) { cl = cl.setId(result); @@ -191,23 +189,23 @@ private void ofLink(String fullLink) throws IOException { cl = cl.setForUsername(result); } - ChannelListResponse clr = cl.execute(); + final ChannelListResponse clr = cl.execute(); if (!clr.getItems().isEmpty()) { - Channel item = clr.getItems().get(0); + final Channel item = clr.getItems().get(0); duplicate(new GroupItem(item)); checkForNewChannel(item.getId()); } } else { - PlaylistListResponse pl = youtube.playlists().list("snippet") - .setKey(FXMLSuite.getYouTubeApiKey()) + final PlaylistListResponse pl = youtube.playlists().list("snippet") + .setKey(CommentSuite.getYouTubeApiKey()) .setId(result) .execute(); if (!pl.getItems().isEmpty()) { - Playlist item = pl.getItems().get(0); + final Playlist item = pl.getItems().get(0); duplicate(new GroupItem(item)); @@ -220,15 +218,15 @@ private void ofLink(String fullLink) throws IOException { /** * Makes sure the channel associated with this GroupItem is in the database. */ - private void checkForNewChannel(String channelId) { + private void checkForNewChannel(final String channelId) { if(database.doesChannelNotExist(channelId)) { try { - ChannelListResponse clr = youtube.channels().list("snippet") - .setKey(FXMLSuite.getYouTubeApiKey()) + final ChannelListResponse clr = youtube.channels().list("snippet") + .setKey(CommentSuite.getYouTubeApiKey()) .setId(channelId) .execute(); - YouTubeChannel channel = new YouTubeChannel(clr.getItems().get(0)); + final YouTubeChannel channel = new YouTubeChannel(clr.getItems().get(0)); database.insertChannels(Collections.singletonList(channel)); } catch (IOException e) { diff --git a/src/main/java/io/mattw/youtube/commentsuite/db/GroupItemVideo.java b/src/main/java/io/mattw/youtube/commentsuite/db/GroupItemVideo.java index 5a20ba5..754e7a9 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/db/GroupItemVideo.java +++ b/src/main/java/io/mattw/youtube/commentsuite/db/GroupItemVideo.java @@ -2,10 +2,10 @@ public class GroupItemVideo { - private String gitemId; - private String videoId; + private final String gitemId; + private final String videoId; - public GroupItemVideo(String gitemId, String videoId) { + public GroupItemVideo(final String gitemId, final String videoId) { this.gitemId = gitemId; this.videoId = videoId; } @@ -18,9 +18,9 @@ public String getVideoId() { return videoId; } - public boolean equals(Object o) { + public boolean equals(final Object o) { if (o instanceof GroupItemVideo) { - GroupItemVideo giv = (GroupItemVideo) o; + final GroupItemVideo giv = (GroupItemVideo) o; return giv.gitemId.equals(gitemId) && giv.videoId.equals(videoId); } return false; diff --git a/src/main/java/io/mattw/youtube/commentsuite/db/GroupStats.java b/src/main/java/io/mattw/youtube/commentsuite/db/GroupStats.java index cf77a3b..2be6a65 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/db/GroupStats.java +++ b/src/main/java/io/mattw/youtube/commentsuite/db/GroupStats.java @@ -5,9 +5,6 @@ import java.util.List; import java.util.Map; -/** - * @author mattwright324 - */ public class GroupStats { private long totalViews = 0; @@ -24,13 +21,11 @@ public class GroupStats { private long totalCommentLikes = 0; private long totalGrabbedComments = 0; + private long totalModeratedComments = 0; private Map mostLikedViewers = new LinkedHashMap<>(); private Map mostActiveViewers = new LinkedHashMap<>(); private Map weeklyCommentHistogram = new LinkedHashMap<>(); - public GroupStats() { - } - public long getTotalViews() { return totalViews; } @@ -136,6 +131,14 @@ public void setTotalGrabbedComments(long totalGrabbedComments) { this.totalGrabbedComments = totalGrabbedComments; } + public long getTotalModeratedComments() { + return totalModeratedComments; + } + + public void setTotalModeratedComments(long totalModeratedComments) { + this.totalModeratedComments = totalModeratedComments; + } + public Map getMostLikedViewers() { return mostLikedViewers; } diff --git a/src/main/java/io/mattw/youtube/commentsuite/db/SQLLoader.java b/src/main/java/io/mattw/youtube/commentsuite/db/SQLLoader.java index 4b624af..e0080e7 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/db/SQLLoader.java +++ b/src/main/java/io/mattw/youtube/commentsuite/db/SQLLoader.java @@ -16,7 +16,6 @@ * - Use in NamedParameterStatement (:params) * - String replacement before use in one of above (:order) * - * @author mattwright324 */ public enum SQLLoader { /* DDL SQL Scripts */ @@ -31,6 +30,9 @@ public enum SQLLoader { GROUP_RENAME("dml_rename_group.sql"), INSERT_REPLACE_VIDEOS("dml_insert_replace_videos.sql"), INSERT_IGNORE_COMMENTS("dml_insert_ignore_comments.sql"), + INSERT_IGNORE_MODERATED_COMMENTS("dml_insert_ignore_moderated_comments.sql"), + DELETE_MODERATED_COMMENT("dml_delete_moderated_comment.sql"), + DELETE_COMMENT("dml_delete_comment.sql"), INSERT_IGNORE_CHANNELS("dml_insert_ignore_channels.sql"), INSERT_IGNORE_GITEM_VIDEO("dml_insert_ignore_gitem_video.sql"), DELETE_GROUP("dml_delete_group.sql"), @@ -48,6 +50,7 @@ public enum SQLLoader { GET_GROUPITEMS("dql_get_groupitems.sql"), GET_ALL_COMMENT_IDS_BY_GROUP("dql_get_all_comment_ids_by_group.sql"), DOES_COMMENT_EXIST("dql_does_comment_exist.sql"), + DOES_MODERATED_COMMENT_EXIST("dql_does_moderated_comment_exist.sql"), DOES_VIDEO_EXIST("dql_does_video_exist.sql"), DOES_CHANNEL_EXIST("dql_does_channel_exist.sql"), GET_ALL_VIDEO_IDS_BY_GROUPITEM("dql_get_video_ids_by_groupitem.sql"), @@ -72,6 +75,7 @@ public enum SQLLoader { GET_GROUP_DISABLED_VIDEOS("dql_get_group_disabled_videos.sql"), GET_VIDEO_STATS("dql_get_video_stats.sql"), GET_COMMENT_STATS("dql_get_comment_stats.sql"), + GET_MODERATED_COMMENT_STATS("dql_get_moderated_comment_stats.sql"), GET_UNIQUE_VIEWERS_BY_GROUP("dql_get_unique_viewers_by_group.sql"), ; diff --git a/src/main/java/io/mattw/youtube/commentsuite/db/YType.java b/src/main/java/io/mattw/youtube/commentsuite/db/YType.java index b261d94..67e411b 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/db/YType.java +++ b/src/main/java/io/mattw/youtube/commentsuite/db/YType.java @@ -3,7 +3,6 @@ /** * Type definition for YouTubeObjects * - * @author mattwright324 */ public enum YType { UNKNOWN("Unknown"), diff --git a/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeChannel.java b/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeChannel.java index 34dcbb1..19faf44 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeChannel.java +++ b/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeChannel.java @@ -4,9 +4,6 @@ import com.google.api.services.youtube.model.Comment; import org.apache.commons.text.StringEscapeUtils; -/** - * @author mattwright324 - */ public class YouTubeChannel extends YouTubeObject { /** diff --git a/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeComment.java b/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeComment.java index 0b2eb41..9589192 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeComment.java +++ b/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeComment.java @@ -4,18 +4,17 @@ import com.google.api.services.youtube.model.CommentSnippet; import com.google.api.services.youtube.model.CommentThread; import com.google.api.services.youtube.model.CommentThreadSnippet; -import io.mattw.youtube.commentsuite.FXMLSuite; +import io.mattw.youtube.commentsuite.CommentSuite; +import io.mattw.youtube.commentsuite.refresh.ModerationStatus; import io.mattw.youtube.commentsuite.util.DateUtils; import org.apache.commons.lang3.builder.ReflectionToStringBuilder; import org.apache.commons.text.StringEscapeUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.time.LocalDateTime; import java.util.List; -/** - * @author mattwright324 - */ public class YouTubeComment extends YouTubeObject implements Exportable { public static final Logger logger = LogManager.getLogger(); @@ -23,11 +22,13 @@ public class YouTubeComment extends YouTubeObject implements Exportable { private String commentText; private String channelId; private transient long published; + private transient LocalDateTime publishedDateTime; private String commentDate; private String videoId; private long likes, replyCount; private boolean isReply; private String parentId; + private ModerationStatus moderationStatus; // Field(s) used just for export to make things pretty. private YouTubeChannel author; @@ -47,6 +48,7 @@ public YouTubeComment(Comment item, String videoId) { CommentSnippet snippet = item.getSnippet(); this.commentText = snippet.getTextDisplay(); this.published = snippet.getPublishedAt().getValue(); + this.publishedDateTime = DateUtils.epochMillisToDateTime(published); this.likes = snippet.getLikeCount(); this.parentId = snippet.getParentId(); @@ -74,7 +76,9 @@ public YouTubeComment(CommentThread item) { CommentSnippet tlcSnippet = snippet.getTopLevelComment().getSnippet(); this.commentText = tlcSnippet.getTextDisplay(); this.published = tlcSnippet.getPublishedAt().getValue(); + this.publishedDateTime = DateUtils.epochMillisToDateTime(published); this.likes = tlcSnippet.getLikeCount(); + this.moderationStatus = ModerationStatus.fromApiValue(tlcSnippet.getModerationStatus()); if (tlcSnippet.getAuthorChannelId() != null) { this.channelId = getChannelIdFromObject(tlcSnippet.getAuthorChannelId()); @@ -92,17 +96,38 @@ public YouTubeComment(String commentId, String text, long published, String vide this.commentText = text; this.published = published; + this.publishedDateTime = DateUtils.epochMillisToDateTime(published); this.videoId = videoId; this.channelId = channelId; this.likes = likes; this.replyCount = replies; this.isReply = isReply; this.parentId = parentId; + this.moderationStatus = ModerationStatus.PUBLISHED; + } + + /** + * Constructor used for initialization from the database. + */ + public YouTubeComment(String commentId, String text, long published, String videoId, String channelId, int likes, int replies, boolean isReply, String parentId, String moderationStatus) { + super(commentId, null, null); + this.setTypeId(YType.COMMENT); + + this.commentText = text; + this.published = published; + this.publishedDateTime = DateUtils.epochMillisToDateTime(published); + this.videoId = videoId; + this.channelId = channelId; + this.likes = likes; + this.replyCount = replies; + this.isReply = isReply; + this.parentId = parentId; + this.moderationStatus = ModerationStatus.fromName(moderationStatus); } @Override public void prepForExport() { - commentDate = DateUtils.epochMillisToDateTime(published).toString(); + commentDate = publishedDateTime.toString(); } public String getCommentText() { @@ -113,6 +138,10 @@ public long getPublished() { return published; } + public LocalDateTime getPublishedDateTime() { + return publishedDateTime; + } + public String getVideoId() { return videoId; } @@ -122,7 +151,7 @@ public String getChannelId() { } public YouTubeChannel getChannel() { - return FXMLSuite.getDatabase().getChannel(channelId); + return CommentSuite.getDatabase().getChannel(channelId); } public long getLikes() { @@ -145,6 +174,14 @@ public YouTubeChannel getAuthor() { return author; } + public ModerationStatus getModerationStatus() { + return moderationStatus; + } + + public void setModerationStatus(ModerationStatus moderationStatus) { + this.moderationStatus = moderationStatus; + } + /** * Overwrite channelId as null when set because it will be on the channel object for export. */ diff --git a/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeObject.java b/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeObject.java index 5a0fda3..6347ef9 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeObject.java +++ b/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeObject.java @@ -6,11 +6,12 @@ import javafx.scene.image.Image; import java.io.Serializable; +import java.util.Objects; +import java.util.stream.Stream; /** * Similarities between GroupItem, YouTubeChannel, YouTubeComment, and YouTubeVideo. * - * @author mattwright324 */ public abstract class YouTubeObject implements ImageCache, Serializable { @@ -36,7 +37,7 @@ public YouTubeObject() { * @param title video title, playlist name, channel name * @param thumbUrl url of thumbnail, profile picture */ - public YouTubeObject(String id, String title, String thumbUrl) { + public YouTubeObject(final String id, final String title, final String thumbUrl) { this.id = id; this.title = title; this.thumbUrl = thumbUrl; @@ -49,7 +50,7 @@ public YouTubeObject(String id, String title, String thumbUrl) { * @param title video title, playlist name, channel name * @param thumbUrl url of thumbnail, profile picture */ - public YouTubeObject(ResourceId id, String title, String thumbUrl) { + public YouTubeObject(final ResourceId id, final String title, final String thumbUrl) { this.id = getIdFromResource(id); this.title = title; this.thumbUrl = thumbUrl; @@ -149,7 +150,7 @@ public boolean equals(Object o) { * * @return channelId */ - static String getChannelIdFromObject(Object authorChannelId) { + public static String getChannelIdFromObject(final Object authorChannelId) { if (authorChannelId instanceof ArrayMap) { ArrayMap value = (ArrayMap) authorChannelId; @@ -161,14 +162,10 @@ static String getChannelIdFromObject(Object authorChannelId) { /** * @return id of video, channel, or playlist */ - protected static String getIdFromResource(ResourceId resourceId) { - if (resourceId.getVideoId() != null) { - return resourceId.getVideoId(); - } else if (resourceId.getChannelId() != null) { - return resourceId.getChannelId(); - } else if (resourceId.getPlaylistId() != null) { - return resourceId.getPlaylistId(); - } - return null; + public static String getIdFromResource(final ResourceId resourceId) { + return Stream.of(resourceId.getVideoId(), resourceId.getChannelId(), resourceId.getPlaylistId()) + .filter(Objects::nonNull) + .findFirst() + .orElse(null); } } diff --git a/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeVideo.java b/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeVideo.java index 427c4ab..28b484c 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeVideo.java +++ b/src/main/java/io/mattw/youtube/commentsuite/db/YouTubeVideo.java @@ -10,9 +10,6 @@ import java.math.BigInteger; import java.util.Optional; -/** - * @author mattwright324 - */ public class YouTubeVideo extends YouTubeObject implements Exportable { private static final Logger logger = LogManager.getLogger(); diff --git a/src/main/java/io/mattw/youtube/commentsuite/events/AccountAddEvent.java b/src/main/java/io/mattw/youtube/commentsuite/events/AccountAddEvent.java new file mode 100644 index 0000000..76fca5a --- /dev/null +++ b/src/main/java/io/mattw/youtube/commentsuite/events/AccountAddEvent.java @@ -0,0 +1,17 @@ +package io.mattw.youtube.commentsuite.events; + +import io.mattw.youtube.commentsuite.oauth2.YouTubeAccount; + +public class AccountAddEvent { + + private final YouTubeAccount account; + + public AccountAddEvent(final YouTubeAccount account) { + this.account = account; + } + + public YouTubeAccount getAccount() { + return account; + } + +} diff --git a/src/main/java/io/mattw/youtube/commentsuite/events/AccountDeleteEvent.java b/src/main/java/io/mattw/youtube/commentsuite/events/AccountDeleteEvent.java new file mode 100644 index 0000000..f47ed09 --- /dev/null +++ b/src/main/java/io/mattw/youtube/commentsuite/events/AccountDeleteEvent.java @@ -0,0 +1,17 @@ +package io.mattw.youtube.commentsuite.events; + +import io.mattw.youtube.commentsuite.oauth2.YouTubeAccount; + +public class AccountDeleteEvent { + + private final YouTubeAccount account; + + public AccountDeleteEvent(final YouTubeAccount account) { + this.account = account; + } + + public YouTubeAccount getAccount() { + return account; + } + +} diff --git a/src/main/java/io/mattw/youtube/commentsuite/events/ReplyEvent.java b/src/main/java/io/mattw/youtube/commentsuite/events/ReplyEvent.java new file mode 100644 index 0000000..73c8d03 --- /dev/null +++ b/src/main/java/io/mattw/youtube/commentsuite/events/ReplyEvent.java @@ -0,0 +1,17 @@ +package io.mattw.youtube.commentsuite.events; + +import io.mattw.youtube.commentsuite.fxml.SearchCommentsListItem; + +public class ReplyEvent { + + private final SearchCommentsListItem commentListItem; + + public ReplyEvent(final SearchCommentsListItem comment) { + this.commentListItem = comment; + } + + public SearchCommentsListItem getCommentListItem() { + return commentListItem; + } + +} diff --git a/src/main/java/io/mattw/youtube/commentsuite/events/ShowMoreEvent.java b/src/main/java/io/mattw/youtube/commentsuite/events/ShowMoreEvent.java new file mode 100644 index 0000000..65f8c36 --- /dev/null +++ b/src/main/java/io/mattw/youtube/commentsuite/events/ShowMoreEvent.java @@ -0,0 +1,17 @@ +package io.mattw.youtube.commentsuite.events; + +import io.mattw.youtube.commentsuite.fxml.SearchCommentsListItem; + +public class ShowMoreEvent { + + private final SearchCommentsListItem commentListItem; + + public ShowMoreEvent(final SearchCommentsListItem comment) { + this.commentListItem = comment; + } + + public SearchCommentsListItem getCommentListItem() { + return commentListItem; + } + +} diff --git a/src/main/java/io/mattw/youtube/commentsuite/events/ViewTreeEvent.java b/src/main/java/io/mattw/youtube/commentsuite/events/ViewTreeEvent.java new file mode 100644 index 0000000..9723c3f --- /dev/null +++ b/src/main/java/io/mattw/youtube/commentsuite/events/ViewTreeEvent.java @@ -0,0 +1,17 @@ +package io.mattw.youtube.commentsuite.events; + +import io.mattw.youtube.commentsuite.fxml.SearchCommentsListItem; + +public class ViewTreeEvent { + + private final SearchCommentsListItem commentListItem; + + public ViewTreeEvent(final SearchCommentsListItem comment) { + this.commentListItem = comment; + } + + public SearchCommentsListItem getCommentListItem() { + return commentListItem; + } + +} diff --git a/src/main/java/io/mattw/youtube/commentsuite/fxml/Cleanable.java b/src/main/java/io/mattw/youtube/commentsuite/fxml/Cleanable.java index d7ae882..f259ce3 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/fxml/Cleanable.java +++ b/src/main/java/io/mattw/youtube/commentsuite/fxml/Cleanable.java @@ -6,7 +6,6 @@ *

* Most applicable to modals. * - * @author mattwright324 */ public interface Cleanable { diff --git a/src/main/java/io/mattw/youtube/commentsuite/fxml/MGCreateGroupModal.java b/src/main/java/io/mattw/youtube/commentsuite/fxml/MGCreateGroupModal.java index 1a91ded..9278dd6 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/fxml/MGCreateGroupModal.java +++ b/src/main/java/io/mattw/youtube/commentsuite/fxml/MGCreateGroupModal.java @@ -14,7 +14,6 @@ /** * This modal allows the user to create a new and empty Group with the name of their choosing. The name must be unique. * - * @author mattwright324 * @see ManageGroups */ public class MGCreateGroupModal extends VBox implements Cleanable { diff --git a/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVAddItemModal.java b/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVAddItemModal.java index 480c390..61d8f35 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVAddItemModal.java +++ b/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVAddItemModal.java @@ -2,7 +2,7 @@ import io.mattw.youtube.commentsuite.ConfigData; import io.mattw.youtube.commentsuite.ConfigFile; -import io.mattw.youtube.commentsuite.FXMLSuite; +import io.mattw.youtube.commentsuite.CommentSuite; import io.mattw.youtube.commentsuite.db.CommentDatabase; import io.mattw.youtube.commentsuite.db.Group; import io.mattw.youtube.commentsuite.db.GroupItem; @@ -27,7 +27,6 @@ * This modal allows the user to add a GroupItem to the Group of the ManageGroupsManager with a YouTube link. * The YouTube link can be any of a video, channel, or playlist and should match the example formats to be accepted. * - * @author mattwright324 * @see GroupItem#GroupItem(String) * @see ManageGroupsManager */ @@ -35,32 +34,27 @@ public class MGMVAddItemModal extends VBox implements Cleanable { private static final Logger logger = LogManager.getLogger(); - private CommentDatabase database; - @FXML private Label alertError; - @FXML private TabPane tabPane; @FXML private Tab tabSingular, tabBulk; @FXML private CheckBox fastGroupAdd; - @FXML private VBox singularPane; @FXML private TextField link; @FXML private Label link1, link2, link3, link4, link5; - @FXML private VBox bulkPane; @FXML private TextArea multiLink; - @FXML private Button btnClose, btnSubmit; - private Group group; + private final Group group; - private ConfigFile configFile = FXMLSuite.getConfig(); - private ConfigData configData = configFile.getDataObject(); + private final ConfigFile configFile = CommentSuite.getConfig(); + private final ConfigData configData = configFile.getDataObject(); + private final CommentDatabase database; - public MGMVAddItemModal(Group group) { + public MGMVAddItemModal(final Group group) { this.group = group; - database = FXMLSuite.getDatabase(); + database = CommentSuite.getDatabase(); FXMLLoader loader = new FXMLLoader(getClass().getResource("MGMVAddItemModal.fxml")); loader.setController(this); diff --git a/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVDeleteGroupModal.java b/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVDeleteGroupModal.java index 91dd739..bbc6d6e 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVDeleteGroupModal.java +++ b/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVDeleteGroupModal.java @@ -15,7 +15,6 @@ * vacuum option is selected before deletion. Alternatively, the user could do a manual vacuum from {@link Settings} * if they did not select vacuum when deleting. * - * @author mattwright324 * @see ManageGroupsManager */ public class MGMVDeleteGroupModal extends VBox { diff --git a/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVGroupItemView.java b/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVGroupItemView.java index 870042e..265ec21 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVGroupItemView.java +++ b/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVGroupItemView.java @@ -16,9 +16,6 @@ import static javafx.application.Platform.runLater; -/** - * @author mattwright324 - */ public class MGMVGroupItemView extends HBox { private GroupItem groupItem; @@ -30,7 +27,7 @@ public class MGMVGroupItemView extends HBox { private BrowserUtil browserUtil = new BrowserUtil(); - public MGMVGroupItemView(GroupItem groupItem) { + public MGMVGroupItemView(final GroupItem groupItem) { this.groupItem = groupItem; FXMLLoader loader = new FXMLLoader(getClass().getResource("MGMVGroupItemView.fxml")); diff --git a/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVRefreshModal.java b/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVRefreshModal.java index 49167c0..67da872 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVRefreshModal.java +++ b/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVRefreshModal.java @@ -1,32 +1,41 @@ package io.mattw.youtube.commentsuite.fxml; -import io.mattw.youtube.commentsuite.*; -import io.mattw.youtube.commentsuite.refresh.*; +import io.mattw.youtube.commentsuite.ConfigData; +import io.mattw.youtube.commentsuite.ConfigFile; +import io.mattw.youtube.commentsuite.CommentSuite; +import io.mattw.youtube.commentsuite.ImageLoader; import io.mattw.youtube.commentsuite.db.Group; +import io.mattw.youtube.commentsuite.refresh.*; import io.mattw.youtube.commentsuite.util.ClipboardUtil; +import io.mattw.youtube.commentsuite.util.Threads; import javafx.beans.binding.Bindings; import javafx.collections.FXCollections; import javafx.collections.ListChangeListener; import javafx.fxml.FXML; import javafx.fxml.FXMLLoader; +import javafx.geometry.Pos; import javafx.scene.control.*; import javafx.scene.image.ImageView; import javafx.scene.input.KeyCode; +import javafx.scene.layout.GridPane; import javafx.scene.layout.HBox; +import javafx.scene.layout.Priority; import javafx.scene.layout.VBox; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.io.IOException; +import java.util.HashMap; +import java.util.Map; -import static io.mattw.youtube.commentsuite.refresh.RefreshStyle.*; +import static io.mattw.youtube.commentsuite.refresh.RefreshStyle.CUSTOM; +import static io.mattw.youtube.commentsuite.refresh.RefreshStyle.values; import static javafx.application.Platform.runLater; /** * This modal allows the user to start a group refresh. The group refresh will use the YouTube API to download * videos under the GroupItems of the Group in the ManageGroupsManager. * - * @author mattwright324 * @see RefreshInterface * @see GroupRefresh * @see ManageGroupsManager @@ -36,8 +45,11 @@ public class MGMVRefreshModal extends HBox { private static final Logger logger = LogManager.getLogger(); public static final int WIDTH = 500; - - private ClipboardUtil clipboard = new ClipboardUtil(); + private static final String STAT_ELAPSED = "STAT_ELAPSED"; + private static final String STAT_VIDEO = "STAT_NEW_VIDEO"; + private static final String STAT_COMMENT = "STAT_NEW_COMMENT"; + private static final String STAT_MODERATED = "STAT_MODERATED"; + private static final String STAT_VIEWER = "STAT_NEW_VIEWER"; @FXML private Label alert; @FXML private Label statusStep; @@ -49,28 +61,34 @@ public class MGMVRefreshModal extends HBox { @FXML private VBox optionsPane; @FXML private ComboBox refreshStyle; @FXML private ComboBox refreshTimeframe; - @FXML private ComboBox refreshCommentPages; + @FXML private ComboBox refreshCommentPages, refreshReviewPages, refreshSpamPages; @FXML private ComboBox refreshCommentOrder; - @FXML private ComboBox refreshReplyPages; + @FXML private ComboBox refreshReplyPages; + @FXML private HBox reviewOption, spamOption; @FXML private HBox warningsPane; - @FXML private Label warnings, elapsedTime, newVideos, totalVideos, newComments, totalComments, newViewers, totalViewers; + @FXML private Label warnings; + @FXML private GridPane refreshStatsPane; @FXML private ImageView expandIcon; @FXML private ListView errorList; @FXML private Hyperlink expand; @FXML private ImageView endStatus; @FXML private ProgressIndicator statusIndicator; - private Group group; + private final Group group; private RefreshInterface refreshThread; private boolean running = false; private boolean hasBeenStarted = false; private boolean expanded = false; - private ConfigFile configFile = FXMLSuite.getConfig(); - private ConfigData configData = configFile.getDataObject(); + private final Map statNewValue = new HashMap<>(); + private final Map statTotalValue = new HashMap<>(); - public MGMVRefreshModal(Group group) { + private final ClipboardUtil clipboard = new ClipboardUtil(); + private final ConfigFile configFile = CommentSuite.getConfig(); + private final ConfigData configData = configFile.getDataObject(); + + public MGMVRefreshModal(final Group group) { logger.debug("Initialize for Group [id={},name={}]", group.getGroupId(), group.getName()); this.group = group; @@ -89,8 +107,12 @@ public MGMVRefreshModal(Group group) { refreshStyle.setItems(FXCollections.observableArrayList(values())); refreshTimeframe.setItems(FXCollections.observableArrayList(RefreshTimeframe.values())); refreshCommentPages.setItems(FXCollections.observableArrayList(RefreshCommentPages.values())); + refreshReviewPages.setItems(FXCollections.observableArrayList(RefreshCommentPages.values())); + refreshSpamPages.setItems(FXCollections.observableArrayList(RefreshCommentPages.values())); refreshCommentOrder.setItems(FXCollections.observableArrayList(RefreshCommentOrder.values())); - refreshReplyPages.setItems(FXCollections.observableArrayList(RefreshReplyPages.values())); + refreshReplyPages.setItems(FXCollections.observableArrayList(RefreshCommentPages.values())); + refreshReviewPages.setValue(RefreshCommentPages.ALL); + refreshSpamPages.setValue(RefreshCommentPages.ALL); refreshStyle.getSelectionModel().selectedItemProperty().addListener((o, ov, nv) -> { logger.debug("Style {}", nv); @@ -109,6 +131,8 @@ public MGMVRefreshModal(Group group) { refreshStyle.setValue(CUSTOM); refreshTimeframe.setValue(refreshOptions.getTimeframe()); refreshCommentPages.setValue(refreshOptions.getCommentPages()); + refreshReviewPages.setValue(refreshOptions.getReviewPages()); + refreshSpamPages.setValue(refreshOptions.getSpamPages()); refreshCommentOrder.setValue(refreshOptions.getCommentOrder()); refreshReplyPages.setValue(refreshOptions.getReplyPages()); } else { @@ -150,10 +174,7 @@ public MGMVRefreshModal(Group group) { }); refreshThread.hardShutdown(); while (refreshThread.isAlive()) { - try { - Thread.sleep(97); - } catch (Exception ignored) { - } + Threads.awaitMillis(97); } running = false; runLater(() -> btnClose.setDisable(false)); @@ -175,6 +196,8 @@ public MGMVRefreshModal(Group group) { options.setStyle(refreshStyle.getValue()); options.setTimeframe(refreshTimeframe.getValue()); options.setCommentPages(refreshCommentPages.getValue()); + options.setReviewPages(refreshReviewPages.getValue()); + options.setSpamPages(refreshSpamPages.getValue()); options.setReplyPages(refreshReplyPages.getValue()); configData.setRefreshOptions(options); @@ -183,31 +206,49 @@ public MGMVRefreshModal(Group group) { refreshThread = new GroupRefresh(group, options); runLater(() -> { + int rowIndex = 1; + createGridRow(STAT_ELAPSED, rowIndex++, "Elapsed time"); + createGridRowNewTotal(STAT_VIDEO, rowIndex++,"New videos"); + createGridRowNewTotal(STAT_COMMENT, rowIndex++,"New comments"); + if (configData.isGrabHeldForReview() || configData.isGrabLikelySpam()) { + createGridRowNewTotal(STAT_MODERATED, rowIndex++,"New moderated"); + } + createGridRowNewTotal(STAT_VIEWER, rowIndex++,"New viewers"); + refreshThread.getObservableErrorList().addListener((ListChangeListener) (lcl) -> runLater(() -> { int items = lcl.getList().size(); warningsPane.setManaged(items > 0); warningsPane.setVisible(items > 0); warnings.setText(items + " message(s)"); })); + errorList.setItems(refreshThread.getObservableErrorList()); - elapsedTime.textProperty().bind(refreshThread.elapsedTimeProperty()); + statNewValue.get(STAT_ELAPSED).textProperty().bind(refreshThread.elapsedTimeProperty()); progressBar.progressProperty().bind(refreshThread.progressProperty()); statusStep.textProperty().bind(refreshThread.statusStepProperty()); - newVideos.textProperty().bind(Bindings.format("%,d", refreshThread.newVideosProperty())); - totalVideos.textProperty().bind( + statNewValue.get(STAT_VIDEO).textProperty().bind(Bindings.format("%,d", refreshThread.newVideosProperty())); + statTotalValue.get(STAT_VIDEO).textProperty().bind( Bindings.concat("of ") .concat(Bindings.format("%,d", refreshThread.totalVideosProperty())) .concat(" total")); - newComments.textProperty().bind(Bindings.format("%,d", refreshThread.newCommentsProperty())); - totalComments.textProperty().bind( + statNewValue.get(STAT_COMMENT).textProperty().bind(Bindings.format("%,d", refreshThread.newCommentsProperty())); + statTotalValue.get(STAT_COMMENT).textProperty().bind( Bindings.concat("of ") .concat(Bindings.format("%,d", refreshThread.totalCommentsProperty())) .concat(" total")); - newViewers.textProperty().bind(Bindings.format("%,d", refreshThread.newViewersProperty())); - totalViewers.textProperty().bind( + if (statNewValue.containsKey(STAT_MODERATED)) { + statNewValue.get(STAT_MODERATED).textProperty().bind(Bindings.format("%,d", refreshThread.newModeratedProperty())); + statTotalValue.get(STAT_MODERATED).textProperty().bind( + Bindings.concat("of ") + .concat(Bindings.format("%,d", refreshThread.totalModeratedProperty())) + .concat(" total")); + } + + statNewValue.get(STAT_VIEWER).textProperty().bind(Bindings.format("%,d", refreshThread.newViewersProperty())); + statTotalValue.get(STAT_VIEWER).textProperty().bind( Bindings.concat("of ") .concat(Bindings.format("%,d", refreshThread.totalViewersProperty())) .concat(" total")); @@ -235,6 +276,42 @@ public MGMVRefreshModal(Group group) { } } + private void createGridRow(final String statKey, final int rowNum, final String displayName) { + final Label name = new Label(displayName); + name.getStyleClass().addAll("bold", "font14"); + name.setAlignment(Pos.TOP_RIGHT); + + final Label newValue = new Label("0"); + newValue.setMinWidth(0); + newValue.setPrefWidth(0); + newValue.setMaxWidth(Double.MAX_VALUE); + GridPane.setHgrow(newValue, Priority.ALWAYS); + statNewValue.put(statKey, newValue); + + refreshStatsPane.addRow(rowNum, name, newValue); + } + + private void createGridRowNewTotal(final String statKey, final int rowNum, final String displayName) { + final Label name = new Label(displayName); + name.getStyleClass().addAll("bold", "font14"); + name.setAlignment(Pos.TOP_RIGHT); + + final Label newValue = new Label("0"); + newValue.setMinWidth(0); + newValue.setPrefWidth(0); + newValue.setMaxWidth(Double.MAX_VALUE); + GridPane.setHgrow(newValue, Priority.ALWAYS); + + final Label totalValue = new Label("0 total"); + totalValue.getStyleClass().addAll("textMutedLight", "font14"); + totalValue.setAlignment(Pos.TOP_RIGHT); + + statNewValue.put(statKey, newValue); + statTotalValue.put(statKey, totalValue); + + refreshStatsPane.addRow(rowNum, name, newValue, totalValue); + } + /** * Reset the modal back to its original state when being opened. */ @@ -242,10 +319,17 @@ public void reset() { logger.debug("Resetting state of Refresh Modal"); running = false; hasBeenStarted = false; + statNewValue.clear(); + statTotalValue.clear(); runLater(() -> { if (expanded) { expand.fire(); } + reviewOption.setVisible(configData.isGrabHeldForReview()); + reviewOption.setManaged(configData.isGrabHeldForReview()); + spamOption.setVisible(configData.isGrabLikelySpam()); + spamOption.setManaged(configData.isGrabLikelySpam()); + refreshStatsPane.getChildren().clear(); endStatus.setManaged(false); endStatus.setVisible(false); statusIndicator.setManaged(true); diff --git a/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVRemoveAllModal.java b/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVRemoveAllModal.java index 48b91ed..afff0bf 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVRemoveAllModal.java +++ b/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVRemoveAllModal.java @@ -1,6 +1,6 @@ package io.mattw.youtube.commentsuite.fxml; -import io.mattw.youtube.commentsuite.FXMLSuite; +import io.mattw.youtube.commentsuite.CommentSuite; import io.mattw.youtube.commentsuite.db.CommentDatabase; import io.mattw.youtube.commentsuite.db.Group; import io.mattw.youtube.commentsuite.db.GroupItem; @@ -24,7 +24,6 @@ /** * This modal allows the user to remove all GroupItems from the Group of its ManageGroupManager. * - * @author mattwright324 * @see ManageGroupsManager */ public class MGMVRemoveAllModal extends VBox implements Cleanable { @@ -38,16 +37,16 @@ public class MGMVRemoveAllModal extends VBox implements Cleanable { @FXML private Button btnClose; @FXML private Button btnSubmit; - private Group group; - private ObservableList groupItems; + private final Group group; + private final ObservableList groupItems; - public MGMVRemoveAllModal(Group group, ObservableList groupItems) { + public MGMVRemoveAllModal(final Group group, final ObservableList groupItems) { logger.debug("Initialize for Group [id={},name={}]", group.getGroupId(), group.getName()); this.group = group; this.groupItems = groupItems; - database = FXMLSuite.getDatabase(); + database = CommentSuite.getDatabase(); FXMLLoader loader = new FXMLLoader(getClass().getResource("MGMVRemoveAllModal.fxml")); loader.setController(this); diff --git a/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVRemoveSelectedModal.java b/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVRemoveSelectedModal.java index e9d2615..a39851c 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVRemoveSelectedModal.java +++ b/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVRemoveSelectedModal.java @@ -1,6 +1,6 @@ package io.mattw.youtube.commentsuite.fxml; -import io.mattw.youtube.commentsuite.FXMLSuite; +import io.mattw.youtube.commentsuite.CommentSuite; import io.mattw.youtube.commentsuite.db.CommentDatabase; import io.mattw.youtube.commentsuite.db.Group; import io.mattw.youtube.commentsuite.db.GroupItem; @@ -24,7 +24,6 @@ /** * This modal allows the user to remove selected GroupItems from the Group of its ManageGroupsManager. * - * @author mattwright324 * @see ManageGroupsManager */ public class MGMVRemoveSelectedModal extends VBox implements Cleanable { @@ -38,16 +37,16 @@ public class MGMVRemoveSelectedModal extends VBox implements Cleanable { @FXML private Button btnClose; @FXML private Button btnSubmit; - private Group group; - private MultipleSelectionModel selectionModel; + private final Group group; + private final MultipleSelectionModel selectionModel; - public MGMVRemoveSelectedModal(Group group, MultipleSelectionModel selectionModel) { + public MGMVRemoveSelectedModal(final Group group, final MultipleSelectionModel selectionModel) { logger.debug("Initialize for Group [id={},name={}]", group.getGroupId(), group.getName()); this.group = group; this.selectionModel = selectionModel; - database = FXMLSuite.getDatabase(); + database = CommentSuite.getDatabase(); FXMLLoader loader = new FXMLLoader(getClass().getResource("MGMVRemoveSelectedModal.fxml")); loader.setController(this); diff --git a/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVYouTubeObjectItem.java b/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVYouTubeObjectItem.java index 02298b3..b12711c 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVYouTubeObjectItem.java +++ b/src/main/java/io/mattw/youtube/commentsuite/fxml/MGMVYouTubeObjectItem.java @@ -20,9 +20,6 @@ import static javafx.application.Platform.runLater; -/** - * @author mattwright324 - */ public class MGMVYouTubeObjectItem extends HBox implements ImageCache { private static final Logger logger = LogManager.getLogger(); @@ -30,7 +27,7 @@ public class MGMVYouTubeObjectItem extends HBox implements ImageCache { @FXML private ImageView thumbnail; @FXML private Label title, subtitle; - private YouTubeObject object; + private final YouTubeObject object; private Long value; private String subtitleText; private String subtitleSuffix; @@ -38,9 +35,9 @@ public class MGMVYouTubeObjectItem extends HBox implements ImageCache { private boolean isVideo = true; private boolean justSubtitle = false; - private BrowserUtil browserUtil = new BrowserUtil(); + private final BrowserUtil browserUtil = new BrowserUtil(); - public MGMVYouTubeObjectItem(YouTubeVideo video, String subtitle) { + public MGMVYouTubeObjectItem(final YouTubeVideo video, final String subtitle) { this.object = video; this.subtitleText = subtitle; this.justSubtitle = true; @@ -48,11 +45,11 @@ public MGMVYouTubeObjectItem(YouTubeVideo video, String subtitle) { initialize(); } - public MGMVYouTubeObjectItem(YouTubeVideo video, Long value, String subtitleSuffix) { + public MGMVYouTubeObjectItem(final YouTubeVideo video, final Long value, final String subtitleSuffix) { this(video, value, subtitleSuffix, false); } - public MGMVYouTubeObjectItem(YouTubeVideo video, Long value, String subtitleSuffix, boolean commentsDisabled) { + public MGMVYouTubeObjectItem(final YouTubeVideo video, final Long value, final String subtitleSuffix, final boolean commentsDisabled) { this.object = video; this.value = value; this.subtitleSuffix = subtitleSuffix; @@ -61,7 +58,7 @@ public MGMVYouTubeObjectItem(YouTubeVideo video, Long value, String subtitleSuff initialize(); } - public MGMVYouTubeObjectItem(YouTubeChannel channel, Long value, String subtitleSuffix) { + public MGMVYouTubeObjectItem(final YouTubeChannel channel, final Long value, final String subtitleSuffix) { this.object = channel; this.value = value; this.subtitleSuffix = subtitleSuffix; diff --git a/src/main/java/io/mattw/youtube/commentsuite/fxml/Main.java b/src/main/java/io/mattw/youtube/commentsuite/fxml/Main.java index 969f0c1..dbc2fc3 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/fxml/Main.java +++ b/src/main/java/io/mattw/youtube/commentsuite/fxml/Main.java @@ -23,7 +23,6 @@ /** * Controls the header: switching content and opening the settings view. * - * @author mattwright324 */ public class Main implements Initializable { diff --git a/src/main/java/io/mattw/youtube/commentsuite/fxml/ManageGroups.java b/src/main/java/io/mattw/youtube/commentsuite/fxml/ManageGroups.java index 3202767..5066b6e 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/fxml/ManageGroups.java +++ b/src/main/java/io/mattw/youtube/commentsuite/fxml/ManageGroups.java @@ -3,7 +3,7 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.google.common.eventbus.Subscribe; -import io.mattw.youtube.commentsuite.FXMLSuite; +import io.mattw.youtube.commentsuite.CommentSuite; import io.mattw.youtube.commentsuite.ImageLoader; import io.mattw.youtube.commentsuite.db.CommentDatabase; import io.mattw.youtube.commentsuite.db.Group; @@ -32,7 +32,6 @@ /** * Manages group selection, creation, and content switching. * - * @author mattwright324 */ public class ManageGroups implements Initializable { @@ -55,9 +54,9 @@ public void initialize(URL location, ResourceBundle resources) { instance = this; - FXMLSuite.getEventBus().register(this); + CommentSuite.getEventBus().register(this); - database = FXMLSuite.getDatabase(); + database = CommentSuite.getDatabase(); /* * Logic for main pane. @@ -165,7 +164,6 @@ public void groupAddEvent(final GroupAddEvent addEvent) { @Subscribe public void groupRenameEvent(final GroupRenameEvent renameEvent) { logger.debug("Group Rename Event"); - //runLater(this::refreshGroupSelect); runLater(this::rebuildGroupSelect); } diff --git a/src/main/java/io/mattw/youtube/commentsuite/fxml/ManageGroupsManager.java b/src/main/java/io/mattw/youtube/commentsuite/fxml/ManageGroupsManager.java index 39d9f37..901bdfd 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/fxml/ManageGroupsManager.java +++ b/src/main/java/io/mattw/youtube/commentsuite/fxml/ManageGroupsManager.java @@ -1,7 +1,10 @@ package io.mattw.youtube.commentsuite.fxml; import com.google.common.eventbus.Subscribe; -import io.mattw.youtube.commentsuite.*; +import io.mattw.youtube.commentsuite.ConfigData; +import io.mattw.youtube.commentsuite.CommentSuite; +import io.mattw.youtube.commentsuite.ImageCache; +import io.mattw.youtube.commentsuite.ImageLoader; import io.mattw.youtube.commentsuite.db.CommentDatabase; import io.mattw.youtube.commentsuite.db.Group; import io.mattw.youtube.commentsuite.db.GroupItem; @@ -14,12 +17,15 @@ import javafx.collections.ListChangeListener; import javafx.fxml.FXML; import javafx.fxml.FXMLLoader; +import javafx.geometry.Pos; import javafx.scene.Node; import javafx.scene.chart.LineChart; import javafx.scene.control.*; import javafx.scene.image.Image; import javafx.scene.image.ImageView; import javafx.scene.input.KeyCode; +import javafx.scene.layout.GridPane; +import javafx.scene.layout.Priority; import javafx.scene.layout.Region; import javafx.scene.layout.StackPane; import javafx.scene.text.Font; @@ -48,21 +54,33 @@ *

* Loads template FXML and displays info from database. * - * @author mattwright324 */ public class ManageGroupsManager extends StackPane implements ImageCache, Cleanable { + private static final Logger logger = LogManager.getLogger(); private static final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("MMM dd, yy"); - private static final Logger logger = LogManager.getLogger(); private final Image edit = ImageLoader.PENCIL.getImage(); private final Image close = ImageLoader.CLOSE.getImage(); private final Image save = ImageLoader.SAVE.getImage(); + private static final String STAT_TOTAL_COMMENTS = "STAT_TOTAL_COMMENTS"; + private static final String STAT_GRABBED_COMMENTS = "STAT_GRABBED_COMMENTS"; + private static final String STAT_MODERATED_COMMENTS = "STAT_MODERATED_COMMENTS"; + private static final String STAT_TOTAL_LIKES = "STAT_TOTAL_LIKES"; + private static final String STAT_UNIQUE_VIEWERS = "STAT_UNIQUE_VIEWERS"; + private static final String STAT_TOTAL_VIDEOS = "STAT_TOTAL_VIDEOS"; + private static final String STAT_TOTAL_VIEWS = "STAT_TOTAL_VIEWS"; + private static final String STAT_VIDEO_LIKES = "STAT_VIDEO_LIKES"; + private static final String STAT_VIDEO_DISLIKES = "STAT_VIDEO_DISLIKES"; + private static final String STAT_LIKE_RATIO = "STAT_LIKE_RATIO"; + private static final String STAT_RATIO_NORM = "STAT_RATIO_NORM"; + private final Group group; private final CommentDatabase database; private final ConfigData configData; + private final Map statLabel = new HashMap<>(); private ChangeListener fontListener; @FXML private OverlayModal refreshModal; @@ -85,10 +103,11 @@ public class ManageGroupsManager extends StackPane implements ImageCache, Cleana @FXML private LineChart commentsLineChart, videosLineChart; private LineChart.Series commentsLineChartData, videosLineChartData; - @FXML private Label totalComments, grabbedComments, totalLikes, totalViewers, totalVideos, totalViews, totalVideoLikes, totalVideoDislikes, - likeDislikeRatio, normalizedRatio; +// @FXML private Label totalComments, grabbedComments, totalLikes, totalViewers, totalVideos, totalViews, totalVideoLikes, totalVideoDislikes, +// likeDislikeRatio, normalizedRatio; @FXML private ListView popularVideosList, dislikedVideosList, commentedVideosList, disabledVideosList, popularViewersList, activeViewersList; + @FXML private GridPane commentStatPane, videoStatPane; @FXML private Accordion accordion; @FXML private TitledPane generalPane, videoPane, viewerPane; @@ -96,10 +115,10 @@ public class ManageGroupsManager extends StackPane implements ImageCache, Cleana public ManageGroupsManager(final Group group) throws IOException { logger.debug("Initialize for Group [id={},name={}]", group.getGroupId(), group.getName()); - database = FXMLSuite.getDatabase(); - configData = FXMLSuite.getConfig().getDataObject(); + database = CommentSuite.getDatabase(); + configData = CommentSuite.getConfig().getDataObject(); - FXMLSuite.getEventBus().register(this); + CommentSuite.getEventBus().register(this); this.group = group; @@ -122,6 +141,22 @@ public ManageGroupsManager(final Group group) throws IOException { videosLineChartData = new LineChart.Series<>(); videosLineChart.getData().add(videosLineChartData); + int cIndex = 1, vIndex = 1; + createGridRow(commentStatPane, STAT_TOTAL_COMMENTS, cIndex++, "Total Comments"); + createGridRow(commentStatPane, STAT_GRABBED_COMMENTS, cIndex++, "Grabbed Comments"); + createGridRow(commentStatPane, STAT_MODERATED_COMMENTS, cIndex++, "Moderated Comments"); + createGridRow(commentStatPane, STAT_TOTAL_LIKES, cIndex++, "Total Likes"); + statLabel.get(STAT_TOTAL_LIKES).setStyle("-fx-text-fill:cornflowerblue"); + createGridRow(commentStatPane, STAT_UNIQUE_VIEWERS, cIndex++, "Unique Viewers"); + createGridRow(videoStatPane, STAT_TOTAL_VIDEOS, vIndex++, "Total Videos"); + createGridRow(videoStatPane, STAT_TOTAL_VIEWS, vIndex++, "Total Views"); + createGridRow(videoStatPane, STAT_VIDEO_LIKES, vIndex++, "Total Video Likes"); + statLabel.get(STAT_VIDEO_LIKES).setStyle("-fx-text-fill:cornflowerblue"); + createGridRow(videoStatPane, STAT_VIDEO_DISLIKES, vIndex++, "Total Video Dislikes"); + statLabel.get(STAT_VIDEO_DISLIKES).setStyle("-fx-text-fill:orangered"); + createGridRow(videoStatPane, STAT_LIKE_RATIO, vIndex++, "Like:Dislike Ratio"); + createGridRow(videoStatPane, STAT_RATIO_NORM, vIndex++, "Normalized Ratio"); + editIcon.setImage(edit); closeIcon.setImage(close); @@ -354,6 +389,18 @@ public ManageGroupsManager(final Group group) throws IOException { mgmvRemoveAll.getBtnClose().setOnAction(ae -> removeAllModal.setVisible(false)); } + private void createGridRow(final GridPane gridPane, final String statKey, final int rowNum, final String displayName) { + final Label name = new Label(displayName); + name.getStyleClass().addAll("bold"); + name.setAlignment(Pos.TOP_RIGHT); + + final Label value = new Label("..."); + statLabel.put(statKey, value); + + gridPane.add(name, 1, rowNum); + gridPane.add(value, 2, rowNum); + } + /** * Reloads displayed timestamp and group stats information. Information displayed is queried from the database * and formatted and processed before being added to the labels, lists, and charts. @@ -478,20 +525,21 @@ public void run() { runLater(() -> { commentsLineChartData.getData().addAll(commentChartData); - totalComments.setText(String.format("%,d", comments)); - grabbedComments.setText(String.format("%,d (%,.2f%%)", grabbed, percentGrabbed)); - totalLikes.setText(String.format("+%,d", groupStats.getTotalCommentLikes())); - totalViewers.setText(String.format("%,d", groupStats.getUniqueViewers())); + statLabel.get(STAT_TOTAL_COMMENTS).setText(String.format("%,d", comments)); + statLabel.get(STAT_GRABBED_COMMENTS).setText(String.format("%,d (%,.2f%%)", grabbed, percentGrabbed)); + statLabel.get(STAT_MODERATED_COMMENTS).setText(String.format("%,d", groupStats.getTotalModeratedComments())); + statLabel.get(STAT_TOTAL_LIKES).setText(String.format("+%,d", groupStats.getTotalCommentLikes())); + statLabel.get(STAT_UNIQUE_VIEWERS).setText(String.format("%,d", groupStats.getUniqueViewers())); videosLineChartData.getData().addAll(videoChartData); - totalVideos.setText(String.format("%,d", groupStats.getTotalVideos())); - totalViews.setText(String.format("%,d", groupStats.getTotalViews())); - totalVideoLikes.setText(String.format("+%,d", groupStats.getTotalLikes())); - totalVideoDislikes.setText(String.format("-%,d", groupStats.getTotalDislikes())); - likeDislikeRatio.setText(String.format("+%,d : -%,d", gcdLikes, gcdDislikes)); - likeDislikeRatio.setStyle(String.format("-fx-text-fill:%s", gcdStyle)); - normalizedRatio.setText(String.format("+%,d : -%,d", nLikes, nDislikes)); - normalizedRatio.setStyle(String.format("-fx-text-fill:%s", gcdStyle)); + statLabel.get(STAT_TOTAL_VIDEOS).setText(String.format("%,d", groupStats.getTotalVideos())); + statLabel.get(STAT_TOTAL_VIEWS).setText(String.format("%,d", groupStats.getTotalViews())); + statLabel.get(STAT_VIDEO_LIKES).setText(String.format("+%,d", groupStats.getTotalLikes())); + statLabel.get(STAT_VIDEO_DISLIKES).setText(String.format("-%,d", groupStats.getTotalDislikes())); + statLabel.get(STAT_LIKE_RATIO).setText(String.format("+%,d : -%,d", gcdLikes, gcdDislikes)); + statLabel.get(STAT_LIKE_RATIO).setStyle(String.format("-fx-text-fill:%s", gcdStyle)); + statLabel.get(STAT_RATIO_NORM).setText(String.format("+%,d : -%,d", nLikes, nDislikes)); + statLabel.get(STAT_RATIO_NORM).setStyle(String.format("-fx-text-fill:%s", gcdStyle)); generalPane.setDisable(false); popularVideosList.getItems().addAll(popularVideos); @@ -610,9 +658,9 @@ public void cleanUp() { runLater(() -> { commentsLineChartData.getData().clear(); videosLineChartData.getData().clear(); - Stream.of(totalComments, grabbedComments, totalLikes, totalVideos, totalViews, totalVideoLikes, totalVideoDislikes, - likeDislikeRatio, normalizedRatio) - .forEach(label -> label.setText("...")); + + statLabel.values().forEach(label -> label.setText("...")); + Stream.of(generalPane, videoPane, viewerPane) .forEach(pane -> pane.setDisable(true)); Stream.of(popularVideosList, dislikedVideosList, commentedVideosList, diff --git a/src/main/java/io/mattw/youtube/commentsuite/fxml/OverlayModal.java b/src/main/java/io/mattw/youtube/commentsuite/fxml/OverlayModal.java index c63f0d5..ca98a42 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/fxml/OverlayModal.java +++ b/src/main/java/io/mattw/youtube/commentsuite/fxml/OverlayModal.java @@ -13,7 +13,6 @@ * Base template for modal format overlaying parent StackPane. * * @param Custom modal content fxml controller class - * @author mattwright324 */ public class OverlayModal extends StackPane { @@ -36,16 +35,16 @@ public String getTitle() { return this.title.getText(); } - public void setTitle(String title) { + public void setTitle(final String title) { this.title.setText(title); } - public void setContent(T content) { + public void setContent(final T content) { this.content.getChildren().clear(); this.content.getChildren().add(content); } - public void setDividerClass(String cssClass) { + public void setDividerClass(final String cssClass) { divider.getStyleClass().clear(); divider.getStyleClass().addAll("divider", cssClass); } @@ -71,7 +70,7 @@ public Label getBottomSpacer() { * * @param show show/hide the spacers */ - void showSpacers(boolean show) { + void showSpacers(final boolean show) { topSpacer.setVisible(show); topSpacer.setManaged(show); bottomSpacer.setVisible(show); diff --git a/src/main/java/io/mattw/youtube/commentsuite/fxml/SCExportModal.java b/src/main/java/io/mattw/youtube/commentsuite/fxml/SCExportModal.java index 2b37fba..97818b6 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/fxml/SCExportModal.java +++ b/src/main/java/io/mattw/youtube/commentsuite/fxml/SCExportModal.java @@ -3,10 +3,11 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonParser; -import io.mattw.youtube.commentsuite.FXMLSuite; +import io.mattw.youtube.commentsuite.CommentSuite; import io.mattw.youtube.commentsuite.ImageCache; import io.mattw.youtube.commentsuite.db.CommentDatabase; import io.mattw.youtube.commentsuite.db.CommentQuery; +import io.mattw.youtube.commentsuite.util.Threads; import javafx.fxml.FXML; import javafx.fxml.FXMLLoader; import javafx.scene.control.*; @@ -36,7 +37,6 @@ * videoId2-comments.json * ... * - * @author mattwright324 * @see SearchComments */ public class SCExportModal extends VBox implements Cleanable, ImageCache { @@ -82,7 +82,7 @@ public class SCExportModal extends VBox implements Cleanable, ImageCache { public SCExportModal() { logger.debug("Initialize SCExportModal"); - database = FXMLSuite.getDatabase(); + database = CommentSuite.getDatabase(); FXMLLoader loader = new FXMLLoader(getClass().getResource("SCExportModal.fxml")); loader.setController(this); @@ -155,10 +155,7 @@ private void startExport() { final double progress = exportProducer.getTotalProcessed().get() / (exportProducer.getTotalAccepted().get() * 1d); runLater(() -> exportProgress.setProgress(progress)); - try { - Thread.sleep(100); - } catch (Exception ignored) { - } + Threads.awaitMillis(100); } if (!exportProducer.isHardShutdown()) { diff --git a/src/main/java/io/mattw/youtube/commentsuite/fxml/SCExportProducer.java b/src/main/java/io/mattw/youtube/commentsuite/fxml/SCExportProducer.java index d310126..0dc241f 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/fxml/SCExportProducer.java +++ b/src/main/java/io/mattw/youtube/commentsuite/fxml/SCExportProducer.java @@ -3,7 +3,7 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.stream.JsonWriter; -import io.mattw.youtube.commentsuite.FXMLSuite; +import io.mattw.youtube.commentsuite.CommentSuite; import io.mattw.youtube.commentsuite.db.*; import io.mattw.youtube.commentsuite.refresh.ConsumerMultiProducer; import io.mattw.youtube.commentsuite.util.ExecutorGroup; @@ -39,7 +39,7 @@ public class SCExportProducer extends ConsumerMultiProducer { public SCExportProducer(final CommentQuery commentQuery, final boolean condensed) { this.commentQuery = commentQuery; this.condensedMode = condensed; - this.database = FXMLSuite.getDatabase(); + this.database = CommentSuite.getDatabase(); this.thisExportFolder = new File(EXPORT_FOLDER, formatter.format(LocalDateTime.now()) + "/"); this.thisExportFolder.mkdirs(); @@ -129,7 +129,7 @@ private void createCommentsFile(final YouTubeVideo video) { comment.prepForExport(); if (condensedMode && comment.getReplyCount() > 0 && !comment.isReply()) { - final List replyList = database.getCommentTree(comment.getId()); + final List replyList = database.getCommentTree(comment.getId(), false); for (final YouTubeComment reply : replyList) { reply.setAuthor(database.getChannel(reply.getChannelId())); reply.prepForExport(); diff --git a/src/main/java/io/mattw/youtube/commentsuite/fxml/SCShowMoreModal.java b/src/main/java/io/mattw/youtube/commentsuite/fxml/SCShowMoreModal.java index b570dcc..8e2af5a 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/fxml/SCShowMoreModal.java +++ b/src/main/java/io/mattw/youtube/commentsuite/fxml/SCShowMoreModal.java @@ -1,10 +1,16 @@ package io.mattw.youtube.commentsuite.fxml; import com.google.api.services.youtube.model.Comment; +import com.google.api.services.youtube.model.CommentSnippet; +import com.google.common.eventbus.Subscribe; import io.mattw.youtube.commentsuite.*; import io.mattw.youtube.commentsuite.db.CommentDatabase; import io.mattw.youtube.commentsuite.db.YouTubeChannel; import io.mattw.youtube.commentsuite.db.YouTubeComment; +import io.mattw.youtube.commentsuite.events.AccountAddEvent; +import io.mattw.youtube.commentsuite.events.AccountDeleteEvent; +import io.mattw.youtube.commentsuite.oauth2.OAuth2Manager; +import io.mattw.youtube.commentsuite.oauth2.YouTubeAccount; import io.mattw.youtube.commentsuite.util.BrowserUtil; import javafx.beans.property.BooleanProperty; import javafx.beans.property.SimpleBooleanProperty; @@ -28,7 +34,6 @@ * in its entirety. It also allows the user to reply to the comment with any of currently signed-into accounts * if they exist. * - * @author mattwright324 * @see SearchComments */ public class SCShowMoreModal extends VBox implements Cleanable, ImageCache { @@ -50,20 +55,22 @@ public class SCShowMoreModal extends VBox implements Cleanable, ImageCache { private YouTubeComment loadedComment; - private CommentDatabase database; - private BrowserUtil browserUtil = new BrowserUtil(); - private OAuth2Handler oAuth2Handler; - private ConfigFile config; - private ConfigData configData; + private final CommentDatabase database; + private final BrowserUtil browserUtil = new BrowserUtil(); + private final OAuth2Manager oAuth2Manager; + private final ConfigFile config; + private final ConfigData configData; public SCShowMoreModal() { logger.debug("Initialize SCShowMoreModal"); - database = FXMLSuite.getDatabase(); - oAuth2Handler = FXMLSuite.getOauth2(); - config = FXMLSuite.getConfig(); + database = CommentSuite.getDatabase(); + oAuth2Manager = CommentSuite.getOauth2Manager(); + config = CommentSuite.getConfig(); configData = config.getDataObject(); + CommentSuite.getEventBus().register(this); + FXMLLoader loader = new FXMLLoader(getClass().getResource("SCShowMoreModal.fxml")); loader.setController(this); loader.setRoot(this); @@ -96,11 +103,7 @@ public SCShowMoreModal() { } }); - configData.accountListChangedProperty().addListener((o, ov, nv) -> runLater(() -> { - comboAccountSelect.getItems().clear(); - comboAccountSelect.getItems().addAll(configData.getAccounts()); - comboAccountSelect.getSelectionModel().select(0); - })); + reloadAccountList(); btnReply.visibleProperty().bind(btnReply.managedProperty()); btnReply.managedProperty().bind(replyPane.visibleProperty()); @@ -117,19 +120,11 @@ public SCShowMoreModal() { }); try { - oAuth2Handler.setTokens(comboAccountSelect.getValue().getTokens()); - - String parentId = loadedComment.isReply() ? + final YouTubeAccount selectedAccount = comboAccountSelect.getValue(); + final String parentId = loadedComment.isReply() ? loadedComment.getParentId() : loadedComment.getId(); - - Comment yourReply = oAuth2Handler.postReply(parentId, replyText.getText()); - - YouTubeComment comment = new YouTubeComment(yourReply, loadedComment.getVideoId()); - - // Update tokens on reply in case we have refreshed them. - // TODO: Better way to refresh tokens on YouTubeAccount and update config? - comboAccountSelect.getValue().setTokens(oAuth2Handler.getTokens()); - config.save(); + final Comment yourReply = postReply(selectedAccount, parentId, replyText.getText()); + final YouTubeComment comment = new YouTubeComment(yourReply, loadedComment.getVideoId()); database.insertComments(Collections.singletonList(comment)); database.commit(); @@ -163,6 +158,22 @@ public SCShowMoreModal() { } } + private void reloadAccountList() { + comboAccountSelect.getItems().clear(); + comboAccountSelect.getItems().addAll(configData.getAccounts()); + comboAccountSelect.getSelectionModel().select(0); + } + + @Subscribe + public void accountAddEvent(final AccountAddEvent accountAddEvent) { + reloadAccountList(); + } + + @Subscribe + public void accountDeleteEvent(final AccountDeleteEvent accountDeleteEvent) { + reloadAccountList(); + } + void setError(String error) { errorMsg.setText(error); errorMsg.setVisible(true); @@ -193,6 +204,47 @@ public void loadComment(YouTubeComment comment, boolean replyMode) { }); } + /** + * Attempts to send a reply to the parent comment id and text supplied. It will attempt to send to reply 5 times + * after failure and throwing an error. On each failure, if it detects the tokens used by the account have + * expired, it will attempt to refresh them and use and newly updated tokens. + * + * @param parentId id of comment or parentId of reply-comment to reply to + * @param textOriginal text to reply to the comment with + * @throws IOException failed to reply + */ + public Comment postReply(final YouTubeAccount account, final String parentId, final String textOriginal) throws IOException { + final CommentSnippet snippet = new CommentSnippet(); + snippet.setParentId(parentId); + snippet.setTextOriginal(textOriginal); + + final Comment comment = new Comment(); + comment.setSnippet(snippet); + + int attempt = 0; + do { + try { + final Comment result = CommentSuite.getYouTube().comments() + .insert("snippet", comment) + .setOauthToken(account.getTokens().getAccessToken()) + .execute(); + + logger.debug("Successfully replied [id={}]", result.getId()); + + return result; + } catch (IOException e) { + logger.warn("Failed on comment reply, {}", e.getLocalizedMessage()); + logger.debug("Refreshing tokens and trying again [attempt={}]", attempt); + + oAuth2Manager.getNewAccessToken(account); + } + + attempt++; + } while (attempt < 5); + + throw new IOException("Could not reply and failed to refresh tokens."); + } + public void enableReplyMode(boolean enable) { replyMode.setValue(enable); } diff --git a/src/main/java/io/mattw/youtube/commentsuite/fxml/SCVideoSelectModal.java b/src/main/java/io/mattw/youtube/commentsuite/fxml/SCVideoSelectModal.java index 9562609..5f5be8d 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/fxml/SCVideoSelectModal.java +++ b/src/main/java/io/mattw/youtube/commentsuite/fxml/SCVideoSelectModal.java @@ -1,6 +1,6 @@ package io.mattw.youtube.commentsuite.fxml; -import io.mattw.youtube.commentsuite.FXMLSuite; +import io.mattw.youtube.commentsuite.CommentSuite; import io.mattw.youtube.commentsuite.ImageLoader; import io.mattw.youtube.commentsuite.db.CommentDatabase; import io.mattw.youtube.commentsuite.db.Group; @@ -32,13 +32,12 @@ * This modal allows the user to select a specific video for comment searching that are within the currently * selected Group and GroupItem prior to opening the modal. * - * @author mattwright324 * @see SearchComments */ public class SCVideoSelectModal extends VBox implements Cleanable { private static final Logger logger = LogManager.getLogger(); - + private static final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("MMM dd, yyyy"); private static final String ALL_VIDEOS = "All Videos"; private CommentDatabase database; @@ -52,17 +51,16 @@ public class SCVideoSelectModal extends VBox implements Cleanable { @FXML private Button btnClose, btnSubmit; private StringProperty valueProperty = new SimpleStringProperty(); + private final Map orderTypes = new LinkedHashMap<>(); private Group group; private GroupItem groupItem; private YouTubeVideo selectedVideo; - private Map orderTypes = new LinkedHashMap<>(); - private DateTimeFormatter formatter = DateTimeFormatter.ofPattern("MMM dd, yyyy"); public SCVideoSelectModal() { logger.debug("Initialize SCVideoSelectModal"); - database = FXMLSuite.getDatabase(); + database = CommentSuite.getDatabase(); orderTypes.put("By Date", "publish_date DESC"); orderTypes.put("By Title", "video_title ASC"); @@ -123,11 +121,11 @@ public Button getBtnSubmit() { return btnSubmit; } - YouTubeVideo getSelectedVideo() { + public YouTubeVideo getSelectedVideo() { return selectedVideo; } - void loadWith(Group group, GroupItem groupItem) { + public void loadWith(Group group, GroupItem groupItem) { if (this.group != group || this.groupItem != groupItem) { this.group = group; this.groupItem = groupItem; @@ -139,7 +137,7 @@ void loadWith(Group group, GroupItem groupItem) { updateVideoList(); } - void updateSelectionLabel() { + public void updateSelectionLabel() { btnReset.setDisable(selectedVideo == null); lblSelection.setText(String.format("%s > %s > %s", group != null ? group.getName() : "$group", @@ -147,7 +145,7 @@ void updateSelectionLabel() { selectedVideo != null ? selectedVideo.getTitle() : ALL_VIDEOS)); } - void updateVideoList() { + public void updateVideoList() { new Thread(() -> { runLater(() -> btnSearch.setDisable(true)); try { @@ -181,7 +179,7 @@ void updateVideoList() { }).start(); } - void setValueProperty(String value) { + public void setValueProperty(String value) { valueProperty.setValue(String.format("Selected: (%s)", value)); } diff --git a/src/main/java/io/mattw/youtube/commentsuite/fxml/SYAddToGroupModal.java b/src/main/java/io/mattw/youtube/commentsuite/fxml/SYAddToGroupModal.java index 5c47a53..48bd8af 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/fxml/SYAddToGroupModal.java +++ b/src/main/java/io/mattw/youtube/commentsuite/fxml/SYAddToGroupModal.java @@ -1,7 +1,7 @@ package io.mattw.youtube.commentsuite.fxml; import com.google.common.eventbus.Subscribe; -import io.mattw.youtube.commentsuite.FXMLSuite; +import io.mattw.youtube.commentsuite.CommentSuite; import io.mattw.youtube.commentsuite.db.CommentDatabase; import io.mattw.youtube.commentsuite.db.Group; import io.mattw.youtube.commentsuite.db.GroupItem; @@ -30,7 +30,6 @@ * This modal allows the user to add selected items (Videos, Channels, Playlists) from the Search YouTube section to an * already existing group or create an entirely new group to add the selection to. * - * @author mattwright324 * @see SearchYouTube */ public class SYAddToGroupModal extends VBox implements Cleanable { @@ -45,18 +44,18 @@ public class SYAddToGroupModal extends VBox implements Cleanable { @FXML private Button btnClose; @FXML private Button btnSubmit; - private CommentDatabase database; + private final CommentDatabase database; - private ListView listView; + private final ListView listView; - public SYAddToGroupModal(ListView listView) { + public SYAddToGroupModal(final ListView listView) { this.listView = listView; logger.debug("Initialize SYAddToGroupModal"); - FXMLSuite.getEventBus().register(this); + CommentSuite.getEventBus().register(this); - database = FXMLSuite.getDatabase(); + database = CommentSuite.getDatabase(); FXMLLoader loader = new FXMLLoader(getClass().getResource("SYAddToGroupModal.fxml")); loader.setController(this); @@ -109,7 +108,7 @@ public SYAddToGroupModal(ListView listView) { } } - private void submitItemsToGroup(List items, Group group) { + private void submitItemsToGroup(final List items, final Group group) { List list = items.stream() .map(SearchYouTubeListItem::getYoutubeURL) .map(link -> { @@ -151,7 +150,7 @@ private void submitItemsToGroup(List items, Group group) } - void setError(String error) { + void setError(final String error) { lblWarn.setText(error); lblWarn.setVisible(true); lblWarn.setManaged(true); diff --git a/src/main/java/io/mattw/youtube/commentsuite/fxml/SearchComments.java b/src/main/java/io/mattw/youtube/commentsuite/fxml/SearchComments.java index 1050931..f83b1a5 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/fxml/SearchComments.java +++ b/src/main/java/io/mattw/youtube/commentsuite/fxml/SearchComments.java @@ -1,10 +1,8 @@ package io.mattw.youtube.commentsuite.fxml; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; import com.google.common.eventbus.Subscribe; import io.mattw.youtube.commentsuite.ConfigData; -import io.mattw.youtube.commentsuite.FXMLSuite; +import io.mattw.youtube.commentsuite.CommentSuite; import io.mattw.youtube.commentsuite.ImageCache; import io.mattw.youtube.commentsuite.ImageLoader; import io.mattw.youtube.commentsuite.db.*; @@ -38,24 +36,15 @@ import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.*; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.Stream; import static javafx.application.Platform.runLater; -/** - * @author mattwright324 - */ public class SearchComments implements Initializable, ImageCache { private static final Logger logger = LogManager.getLogger(); - private Cache videoCache = CacheBuilder.newBuilder() - .maximumSize(500) - .expireAfterAccess(5, TimeUnit.MINUTES) - .build(); - @FXML private VBox contextPane, resultsPane, queryPane; @FXML private ImageView videoThumb, authorThumb, toggleContextIcon, toggleQueryIcon; @FXML private ImageView firstPageIcon, prevPageIcon, nextPageIcon, lastPageIcon; @@ -94,23 +83,22 @@ public class SearchComments implements Initializable, ImageCache { private SimpleIntegerProperty pageProperty = new SimpleIntegerProperty(); private SimpleIntegerProperty maxPageProperty = new SimpleIntegerProperty(); private ElapsedTime elapsedTime = new ElapsedTime(); - private ChangeListener cl; private CommentDatabase database; private CommentQuery commentQuery; private List lastResultsList; - private SearchCommentsListItem actionComment; + private SearchCommentsListItem originalTreeComment; private ClipboardUtil clipboardUtil = new ClipboardUtil(); private BrowserUtil browserUtil = new BrowserUtil(); private ConfigData configData; @Override public void initialize(URL location, ResourceBundle resources) { - database = FXMLSuite.getDatabase(); + database = CommentSuite.getDatabase(); commentQuery = database.commentQuery(); - configData = FXMLSuite.getConfig().getDataObject(); + configData = CommentSuite.getConfig().getDataObject(); - FXMLSuite.getEventBus().register(this); + CommentSuite.getEventBus().register(this); SelectionModel selectionModel = comboGroupSelect.getSelectionModel(); selectionModel.selectedItemProperty().addListener((o, ov, nv) -> { @@ -219,7 +207,7 @@ public void initialize(URL location, ResourceBundle resources) { MultipleSelectionModel scSelection = resultsList.getSelectionModel(); scSelection.selectedItemProperty().addListener((o, ov, nv) -> new Thread(() -> { if (nv != null) { - loadCommentContext(nv); + showListItemContext(nv); } }).start()); @@ -318,7 +306,7 @@ public void initialize(URL location, ResourceBundle resources) { runLater(() -> { Optional toSelect = resultsList.getItems().stream() - .filter(scli -> scli.getComment().getId().equals(actionComment.getComment().getId())) + .filter(scli -> scli.getComment().getId().equals(originalTreeComment.getComment().getId())) .findFirst(); SearchCommentsListItem scli = toSelect.orElse(null); @@ -377,56 +365,50 @@ public void initialize(URL location, ResourceBundle resources) { /** * Load video context and comment author profiles on comment interaction: on selection, show more, reply, view thread */ - private void loadCommentContext(SearchCommentsListItem commentItem) { - YouTubeComment comment = commentItem.getComment(); + private void showListItemContext(final SearchCommentsListItem commentItem) { + final YouTubeComment comment = commentItem.getComment(); commentItem.loadProfileThumb(); - for (SearchCommentsListItem scli : resultsList.getItems()) { + for (final SearchCommentsListItem scli : resultsList.getItems()) { scli.checkProfileThumb(); } try { - String videoId = comment.getVideoId(); - YouTubeVideo video = videoCache.getIfPresent(videoId); - if (video == null) { - video = database.getVideo(videoId); - videoCache.put(videoId, video); - } + final String videoId = comment.getVideoId(); + final YouTubeVideo video = database.getVideo(videoId); - final YouTubeVideo fVideo = video; - final Image fVideoThumb = ImageCache.findOrGetImage(video); + final Image vThumb = ImageCache.findOrGetImage(video); runLater(() -> { - videoTitle.setText(fVideo.getTitle()); - videoLikes.setText(trunc(fVideo.getLikes())); - videoDislikes.setText(trunc(fVideo.getDislikes())); - videoViews.setText(String.format("%s views", trunc(fVideo.getViewCount()))); + videoTitle.setText(video.getTitle()); + videoLikes.setText(readableNumber(video.getLikes())); + videoDislikes.setText(readableNumber(video.getDislikes())); + videoViews.setText(String.format("%s views", readableNumber(video.getViewCount()))); videoDescription.setText(String.format("Published %s • %s", - formatter.format(DateUtils.epochMillisToDateTime(fVideo.getPublishedDate())), - StringEscapeUtils.unescapeHtml4(fVideo.getDescription()))); + formatter.format(DateUtils.epochMillisToDateTime(video.getPublishedDate())), + StringEscapeUtils.unescapeHtml4(video.getDescription()))); videoThumb.setCursor(Cursor.HAND); - videoThumb.setOnMouseClicked(me -> browserUtil.open(fVideo.buildYouTubeLink())); - videoThumb.setImage(fVideoThumb); + videoThumb.setOnMouseClicked(me -> browserUtil.open(video.buildYouTubeLink())); + videoThumb.setImage(vThumb); }); - - final YouTubeChannel fVideoAuthor = database.getChannel(video.getChannelId()); - final Image fAuthorThumb = fVideoAuthor != null ? - ImageCache.findOrGetImage(fVideoAuthor) : ImageCache.toLetterAvatar(' '); + final YouTubeChannel author = database.getChannel(video.getChannelId()); + final Image aThumb = author != null ? + ImageCache.findOrGetImage(author) : ImageCache.toLetterAvatar(' '); runLater(() -> { - if (fVideoAuthor != null) { - author.setText(fVideoAuthor.getTitle()); + if (author != null) { + this.author.setText(author.getTitle()); authorThumb.setCursor(Cursor.HAND); - authorThumb.setOnMouseClicked(me -> browserUtil.open(fVideoAuthor.buildYouTubeLink())); - authorThumb.setImage(fAuthorThumb); + authorThumb.setOnMouseClicked(me -> browserUtil.open(author.buildYouTubeLink())); + authorThumb.setImage(aThumb); } else { - author.setText("Error: Null Channel"); + this.author.setText("Error: Null Channel"); authorThumb.setCursor(Cursor.DEFAULT); authorThumb.setOnMouseClicked(null); - authorThumb.setImage(fAuthorThumb); + authorThumb.setImage(aThumb); - logger.error("Channel for video was null [id={}]", fVideo.getChannelId()); + logger.error("Channel for video was null [id={}]", video.getChannelId()); } }); } catch (SQLException e) { @@ -554,12 +536,6 @@ private void setResultsList(List comments, boolean treeMode) { .filter(Objects::nonNull) .collect(Collectors.toList()); - commentListItems.forEach(comment -> { - comment.getShowMore().setOnAction(ae -> showMore(comment)); - comment.getReply().setOnAction(ae -> reply(comment)); - comment.getViewTree().setOnAction(ae -> viewTree(comment)); - }); - runLater(() -> { resultsList.getItems().clear(); resultsList.getItems().addAll(commentListItems); @@ -581,7 +557,7 @@ private void setResultsList(List comments, boolean treeMode) { resultsList.scrollTo(0); Optional toSelect = resultsList.getItems().stream() - .filter(scli -> scli.getComment().getId().equals(actionComment.getComment().getId())) + .filter(scli -> scli.getComment().getId().equals(originalTreeComment.getComment().getId())) .findFirst(); resultsList.getSelectionModel().select(toSelect.orElse(null)); @@ -589,38 +565,40 @@ private void setResultsList(List comments, boolean treeMode) { }); } - private void selectAndShowContext(SearchCommentsListItem item) { + private void selectListItem(SearchCommentsListItem item) { resultsList.getSelectionModel().clearSelection(); resultsList.getSelectionModel().select(item); } - private void showMore(SearchCommentsListItem scli) { - selectAndShowContext(scli); - - YouTubeComment comment = scli.getComment(); + @Subscribe + public void showMoreEvent(final ShowMoreEvent showMoreEvent) { + final SearchCommentsListItem listItem = showMoreEvent.getCommentListItem(); + final YouTubeComment comment = listItem.getComment(); logger.debug("Showing more window for commment [videoId={},commentId={}]", comment.getVideoId(), comment.getId()); - loadCommentContext(scli); - commentModal(comment, false); + selectListItem(listItem); + openReplyModal(comment, false); + showListItemContext(listItem); } - private void reply(SearchCommentsListItem scli) { - selectAndShowContext(scli); - - YouTubeComment comment = scli.getComment(); + @Subscribe + private void replyEvent(final ReplyEvent replyEvent) { + final SearchCommentsListItem listItem = replyEvent.getCommentListItem(); + final YouTubeComment comment = listItem.getComment(); logger.debug("Showing reply window for commment [videoId={},commentId={}]", comment.getVideoId(), comment.getId()); - loadCommentContext(scli); - commentModal(comment, true); + selectListItem(listItem); + openReplyModal(comment, true); + showListItemContext(listItem); } - private void commentModal(YouTubeComment comment, boolean replyMode) { + private void openReplyModal(final YouTubeComment comment, final boolean replyMode) { runLater(() -> { showMoreModal.setVisible(true); showMoreModal.setManaged(true); @@ -631,28 +609,24 @@ private void commentModal(YouTubeComment comment, boolean replyMode) { }); } - private void viewTree(SearchCommentsListItem scli) { - selectAndShowContext(scli); - - runLater(() -> searchingProperty.setValue(true)); - - actionComment = scli; - - loadCommentContext(scli); - - YouTubeComment comment = scli.getComment(); - - String parentId = comment.isReply() ? comment.getParentId() : comment.getId(); + @Subscribe + public void viewTreeEvent(final ViewTreeEvent viewTreeEvent) { + final SearchCommentsListItem listItem = originalTreeComment = viewTreeEvent.getCommentListItem(); + final YouTubeComment comment = listItem.getComment(); + final String parentId = comment.isReply() ? comment.getParentId() : comment.getId(); logger.debug("Viewing comment reply tree [videoId={},commentId={},parentId={}]", comment.getVideoId(), comment.getId(), parentId); - try { - List comments = database.getCommentTree(parentId); + selectListItem(listItem); + showListItemContext(listItem); - setResultsList(comments, true); + runLater(() -> searchingProperty.setValue(true)); + + try { + setResultsList(database.getCommentTree(parentId, comboCommentType.getValue() == CommentQuery.CommentsType.MODERATED_ONLY), true); } catch (SQLException e) { logger.debug("Failed to view comment tree [commentId={},parentId={}]", comment.getId(), parentId); @@ -676,7 +650,6 @@ public void groupAddEvent(final GroupAddEvent addEvent) { @Subscribe public void groupRenameEvent(final GroupRenameEvent renameEvent) { logger.debug("Group Rename Event"); - //runLater(this::refreshGroupSelect); runLater(this::rebuildGroupSelect); } @@ -718,17 +691,9 @@ private void reloadGroupItems() { comboGroupItemSelect.getSelectionModel().select(0); videoSelect.setDisable(false); }); - } - /** - * Truncates a number with shorthand: - * 2000 -> 2.0k - * 5422000 -> 54.2k - * 123456789 -> 123.5m - * 1234567890 -> 1.2b - */ - public String trunc(double value) { + public String readableNumber(double value) { char[] suffix = new char[]{'k', 'm', 'b', 't'}; int pos = 0; while (value > 1000) { diff --git a/src/main/java/io/mattw/youtube/commentsuite/fxml/SearchCommentsListItem.java b/src/main/java/io/mattw/youtube/commentsuite/fxml/SearchCommentsListItem.java index b4034f1..0370d72 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/fxml/SearchCommentsListItem.java +++ b/src/main/java/io/mattw/youtube/commentsuite/fxml/SearchCommentsListItem.java @@ -1,8 +1,15 @@ package io.mattw.youtube.commentsuite.fxml; -import io.mattw.youtube.commentsuite.*; +import com.google.common.eventbus.EventBus; +import com.google.common.eventbus.Subscribe; +import io.mattw.youtube.commentsuite.ConfigData; +import io.mattw.youtube.commentsuite.CommentSuite; +import io.mattw.youtube.commentsuite.ImageCache; +import io.mattw.youtube.commentsuite.ImageLoader; import io.mattw.youtube.commentsuite.db.YouTubeChannel; import io.mattw.youtube.commentsuite.db.YouTubeComment; +import io.mattw.youtube.commentsuite.events.*; +import io.mattw.youtube.commentsuite.refresh.ModerationStatus; import io.mattw.youtube.commentsuite.util.BrowserUtil; import io.mattw.youtube.commentsuite.util.DateUtils; import javafx.fxml.FXML; @@ -18,15 +25,15 @@ import org.apache.logging.log4j.Logger; import java.io.IOException; +import java.time.LocalDateTime; +import static io.mattw.youtube.commentsuite.refresh.ModerationStatus.PUBLISHED; import static javafx.application.Platform.runLater; -/** - * @author mattwright324 - */ public class SearchCommentsListItem extends HBox implements Cleanable { private static final Logger logger = LogManager.getLogger(); + private static final LocalDateTime DAYS_AGO_60 = LocalDateTime.now().minusDays(60); @FXML private ImageView thumbnail; @FXML private Hyperlink author; @@ -35,25 +42,29 @@ public class SearchCommentsListItem extends HBox implements Cleanable { @FXML private Label type; @FXML private Label likes; @FXML private Hyperlink showMore, viewTree, reply; + @FXML private HBox tagsPane; - private YouTubeComment comment; - private YouTubeChannel channel; + private final BrowserUtil browserUtil = new BrowserUtil(); + private final ConfigData configData; + private final EventBus eventBus; - private BrowserUtil browserUtil = new BrowserUtil(); - private ConfigData configData; + private final YouTubeComment comment; + private final YouTubeChannel channel; + private boolean showReplyBtn = true; - SearchCommentsListItem(YouTubeComment comment) throws IOException { + public SearchCommentsListItem(final YouTubeComment comment) throws IOException { this.comment = comment; + this.channel = comment.getChannel(); - configData = FXMLSuite.getConfig().getDataObject(); + configData = CommentSuite.getConfig().getDataObject(); + eventBus = CommentSuite.getEventBus(); + eventBus.register(this); FXMLLoader loader = new FXMLLoader(getClass().getResource("SearchCommentsListItem.fxml")); loader.setController(this); loader.setRoot(this); loader.load(); - channel = comment.getChannel(); - thumbnail.setImage(channel.getDefaultThumb()); checkProfileThumb(); @@ -74,9 +85,14 @@ public class SearchCommentsListItem extends HBox implements Cleanable { } } + final ModerationStatus status = comment.getModerationStatus(); + if (status != null && status != PUBLISHED) { + this.getStyleClass().add(status.getApiValue()); + } + if (comment.isReply()) { - type.setText("Reply"); this.getStyleClass().add("reply"); + type.setText("Reply"); } if (comment.getLikes() > 0) { @@ -86,33 +102,56 @@ public class SearchCommentsListItem extends HBox implements Cleanable { likes.setManaged(false); } - reply.setManaged(!configData.getAccounts().isEmpty()); - reply.setVisible(!configData.getAccounts().isEmpty()); + if (status != null && status != PUBLISHED) { + addTag(status.getApiValue()); + } - configData.accountListChangedProperty().addListener((o, ov, nv) -> { - reply.setManaged(!configData.getAccounts().isEmpty()); - reply.setVisible(!configData.getAccounts().isEmpty()); - }); - } + if (status != null && status != PUBLISHED && comment.getPublishedDateTime().isBefore(DAYS_AGO_60)) { + reply.setManaged(false); + reply.setVisible(false); + viewTree.setManaged(false); + viewTree.setVisible(false); + showReplyBtn = false; + addTag("past-60-days"); + } + + determineHideReply(); + showMore.setOnAction(ae -> eventBus.post(new ShowMoreEvent(this))); + reply.setOnAction(ae -> eventBus.post(new ReplyEvent(this))); + viewTree.setOnAction(ae -> eventBus.post(new ViewTreeEvent(this))); + } - YouTubeComment getComment() { + public YouTubeComment getComment() { return comment; } - Hyperlink getShowMore() { - return showMore; + public void addTag(String text) { + final Label tag = new Label(text); + tag.getStyleClass().addAll("textMuted", "tag"); + runLater(() -> tagsPane.getChildren().add(tag)); + } + + private void determineHideReply() { + final boolean display = !configData.getAccounts().isEmpty() && showReplyBtn; + + runLater(() -> { + reply.setManaged(display); + reply.setVisible(display); + }); } - Hyperlink getReply() { - return reply; + @Subscribe + public void accountAddEvent(final AccountAddEvent accountAddEvent) { + determineHideReply(); } - Hyperlink getViewTree() { - return viewTree; + @Subscribe + public void accountDeleteEvent(final AccountDeleteEvent accountDeleteEvent) { + determineHideReply(); } - void treeMode() { + public void treeMode() { viewTree.setVisible(false); viewTree.setManaged(false); } @@ -120,7 +159,7 @@ void treeMode() { /** * Loads profile thumbnail. */ - void loadProfileThumb() { + public void loadProfileThumb() { runLater(() -> thumbnail.setImage(ImageLoader.LOADING.getImage())); Image thumbImage = ImageCache.findOrGetImage(channel); runLater(() -> thumbnail.setImage(thumbImage)); @@ -129,7 +168,7 @@ void loadProfileThumb() { /** * Checks if profile thumb loaded and loads if present. */ - void checkProfileThumb() { + public void checkProfileThumb() { if (ImageCache.hasImageCached(channel)) { loadProfileThumb(); } diff --git a/src/main/java/io/mattw/youtube/commentsuite/fxml/SearchYouTube.java b/src/main/java/io/mattw/youtube/commentsuite/fxml/SearchYouTube.java index 2cd4f88..2337d45 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/fxml/SearchYouTube.java +++ b/src/main/java/io/mattw/youtube/commentsuite/fxml/SearchYouTube.java @@ -3,9 +3,12 @@ import com.google.api.services.youtube.YouTube; import com.google.api.services.youtube.model.SearchListResponse; import com.google.api.services.youtube.model.SearchResult; -import io.mattw.youtube.commentsuite.FXMLSuite; +import io.mattw.youtube.commentsuite.CommentSuite; import io.mattw.youtube.commentsuite.ImageLoader; -import io.mattw.youtube.commentsuite.util.*; +import io.mattw.youtube.commentsuite.util.BrowserUtil; +import io.mattw.youtube.commentsuite.util.ClipboardUtil; +import io.mattw.youtube.commentsuite.util.IpApiProvider; +import io.mattw.youtube.commentsuite.util.Location; import javafx.beans.binding.BooleanBinding; import javafx.beans.property.SimpleBooleanProperty; import javafx.collections.ListChangeListener; @@ -28,17 +31,15 @@ import static javafx.application.Platform.runLater; -/** - * @author mattwright324 - */ public class SearchYouTube implements Initializable { private static final Logger logger = LogManager.getLogger(); + private static final String TOKEN_FOO = "TOKEN_FOO"; private Location location; - private YouTube youtubeApi; - private ClipboardUtil clipboardUtil = new ClipboardUtil(); - private BrowserUtil browserUtil = new BrowserUtil(); + private YouTube youTube; + private final ClipboardUtil clipboardUtil = new ClipboardUtil(); + private final BrowserUtil browserUtil = new BrowserUtil(); @FXML private Pane form; @FXML private ImageView searchIcon; @@ -68,8 +69,7 @@ public class SearchYouTube implements Initializable { private int total = 0; private int number = 0; - private String emptyToken = "emptyToken"; - private String pageToken = emptyToken; + private String pageToken = TOKEN_FOO; private YouTube.Search.List searchList; private SimpleBooleanProperty searching = new SimpleBooleanProperty(false); @@ -80,8 +80,8 @@ public class SearchYouTube implements Initializable { public void initialize(URL location, ResourceBundle resources) { logger.debug("Initialize SearchYouTube"); - youtubeApi = FXMLSuite.getYouTube(); - this.location = FXMLSuite.getLocation(); + this.youTube = CommentSuite.getYouTube(); + this.location = CommentSuite.getLocation(); MultipleSelectionModel selectionModel = resultsList.getSelectionModel(); @@ -161,7 +161,7 @@ public void initialize(URL location, ResourceBundle resources) { submit.setOnAction(ae -> { total = 0; number = 0; - pageToken = emptyToken; + pageToken = TOKEN_FOO; resultsList.getItems().clear(); runLater(() -> searchInfo.setText(String.format("Showing %s out of %s", resultsList.getItems().size(), total))); logger.debug("Submit New Search [pageToken={},type={},text={},locText={},locRadius={},order={},result={}]", @@ -204,7 +204,7 @@ public void search(String pageToken, String type, String text, String locText, S String encodedText = URLEncoder.encode(text, "UTF-8"); String searchType = types[resultType]; - if (pageToken.equals(emptyToken)) { + if (pageToken.equals(TOKEN_FOO)) { pageToken = ""; } @@ -214,8 +214,8 @@ public void search(String pageToken, String type, String text, String locText, S order = "viewCount"; } - searchList = youtubeApi.search().list("snippet") - .setKey(FXMLSuite.getYouTubeApiKey()) + searchList = youTube.search().list("snippet") + .setKey(CommentSuite.getYouTubeApiKey()) .setMaxResults(50L) .setPageToken(pageToken) .setQ(encodedText) @@ -225,12 +225,12 @@ public void search(String pageToken, String type, String text, String locText, S SearchListResponse sl; if (type.equals("Normal")) { logger.debug("Normal Search [key={},part=snippet,text={},type={},order={},token={}]", - FXMLSuite.getYouTubeApiKey(), encodedText, searchType, order.toLowerCase(), pageToken); + CommentSuite.getYouTubeApiKey(), encodedText, searchType, order.toLowerCase(), pageToken); sl = searchList.execute(); } else { logger.debug("Location Search [key={},part=snippet,text={},loc={},radius={},type={},order={},token={}]", - FXMLSuite.getYouTubeApiKey(), encodedText, locText, locRadius, searchType, order.toLowerCase(), pageToken); + CommentSuite.getYouTubeApiKey(), encodedText, locText, locRadius, searchType, order.toLowerCase(), pageToken); sl = searchList .setType("video") @@ -239,7 +239,7 @@ public void search(String pageToken, String type, String text, String locText, S .execute(); } - this.pageToken = sl.getNextPageToken() == null ? emptyToken : sl.getNextPageToken(); + this.pageToken = sl.getNextPageToken() == null ? TOKEN_FOO : sl.getNextPageToken(); this.total = sl.getPageInfo().getTotalResults(); logger.debug("Search [videos={}]", sl.getItems().size()); @@ -259,7 +259,7 @@ public void search(String pageToken, String type, String text, String locText, S e.printStackTrace(); } runLater(() -> { - if (this.pageToken != null && !this.pageToken.equals(emptyToken)) { + if (this.pageToken != null && !this.pageToken.equals(TOKEN_FOO)) { btnNextPage.setDisable(false); } searching.setValue(false); diff --git a/src/main/java/io/mattw/youtube/commentsuite/fxml/SearchYouTubeListItem.java b/src/main/java/io/mattw/youtube/commentsuite/fxml/SearchYouTubeListItem.java index f37b959..a070a14 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/fxml/SearchYouTubeListItem.java +++ b/src/main/java/io/mattw/youtube/commentsuite/fxml/SearchYouTubeListItem.java @@ -3,7 +3,6 @@ import com.google.api.services.youtube.model.SearchResult; import io.mattw.youtube.commentsuite.ImageCache; import io.mattw.youtube.commentsuite.ImageLoader; -import io.mattw.youtube.commentsuite.YouTubeSearchItem; import javafx.fxml.FXML; import javafx.fxml.FXMLLoader; import javafx.scene.control.Label; @@ -22,7 +21,6 @@ *

* Searching occurs in the SearchYouTube. * - * @author mattwright324 */ public class SearchYouTubeListItem extends HBox { @@ -37,12 +35,12 @@ public class SearchYouTubeListItem extends HBox { @FXML private Label number; @FXML private ImageView thumbnail; - private SearchResult data; + private final SearchResult data; private String objectId; private String youtubeURL; private String typeStr; - public SearchYouTubeListItem(SearchResult data, int num) throws IOException { + public SearchYouTubeListItem(final SearchResult data, final int num) throws IOException { this.data = data; FXMLLoader loader = new FXMLLoader(getClass().getResource("SearchYouTubeListItem.fxml")); @@ -76,8 +74,8 @@ public SearchYouTubeListItem(SearchResult data, int num) throws IOException { description.setText(data.getSnippet().getDescription()); new Thread(() -> { - YouTubeSearchItem obj = new YouTubeSearchItem(data); - Image thumb = ImageCache.findOrGetImage(obj); + final YouTubeSearchItem obj = new YouTubeSearchItem(data); + final Image thumb = ImageCache.findOrGetImage(obj); thumbnail.setFitWidth(thumbnail.getFitHeight() * thumb.getWidth() / thumb.getHeight()); thumbnail.setImage(thumb); }).start(); @@ -85,7 +83,7 @@ public SearchYouTubeListItem(SearchResult data, int num) throws IOException { title.setText("SearchList.Item Error"); author.setText(data.getEtag()); description.setText("There was no snippet attached to this object."); - Image thumb = ImageLoader.OOPS.getImage(); + final Image thumb = ImageLoader.OOPS.getImage(); runLater(() -> { thumbnail.setFitWidth(thumbnail.getFitHeight() * thumb.getWidth() / thumb.getHeight()); thumbnail.setImage(thumb); diff --git a/src/main/java/io/mattw/youtube/commentsuite/fxml/Settings.java b/src/main/java/io/mattw/youtube/commentsuite/fxml/Settings.java index a9efbb6..1144106 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/fxml/Settings.java +++ b/src/main/java/io/mattw/youtube/commentsuite/fxml/Settings.java @@ -1,7 +1,11 @@ package io.mattw.youtube.commentsuite.fxml; +import com.google.common.eventbus.Subscribe; import io.mattw.youtube.commentsuite.*; import io.mattw.youtube.commentsuite.db.CommentDatabase; +import io.mattw.youtube.commentsuite.events.AccountAddEvent; +import io.mattw.youtube.commentsuite.events.AccountDeleteEvent; +import io.mattw.youtube.commentsuite.oauth2.OAuth2Manager; import io.mattw.youtube.commentsuite.util.BrowserUtil; import javafx.concurrent.Worker; import javafx.fxml.FXML; @@ -27,16 +31,14 @@ import static javafx.application.Platform.runLater; -/** - * @author mattwright324 - */ public class Settings implements Initializable { private static final Logger logger = LogManager.getLogger(); private BrowserUtil browserUtil = new BrowserUtil(); private ConfigFile config; - private OAuth2Handler oauth2; + private ConfigData configData; + private OAuth2Manager oAuth2Manager; private CommentDatabase database; @FXML private Pane settingsPane; @@ -58,6 +60,7 @@ public class Settings implements Initializable { @FXML private TextField youtubeApiKey; @FXML private Button btnAddAccount; @FXML private ListView accountList; + @FXML private CheckBox grabHeldForReview, grabLikelySpam; @FXML private ProgressIndicator cleanProgress; @FXML private Button btnClean; @@ -74,41 +77,40 @@ public class Settings implements Initializable { public void initialize(URL location, ResourceBundle resources) { logger.debug("Initialize Settings"); - oauth2 = FXMLSuite.getOauth2(); - config = FXMLSuite.getConfig(); - database = FXMLSuite.getDatabase(); + database = CommentSuite.getDatabase(); + oAuth2Manager = CommentSuite.getOauth2Manager(); + config = CommentSuite.getConfig(); + configData = config.getDataObject(); + CommentSuite.getEventBus().register(this); - ConfigData configData = config.getDataObject(); - configData.refreshAccounts(); autoLoadStats.setSelected(configData.isAutoLoadStats()); prefixReply.setSelected(configData.isPrefixReplies()); downloadThumbs.setSelected(configData.isArchiveThumbs()); customKey.setSelected(configData.isCustomApiKey()); youtubeApiKey.setText(configData.getYoutubeApiKey()); filterDuplicatesOnCopy.setSelected(configData.isFilterDuplicatesOnCopy()); + grabHeldForReview.setSelected(configData.isGrabHeldForReview()); + grabLikelySpam.setSelected(configData.isGrabLikelySpam()); + + CookieHandler.setDefault(new CookieManager()); - CookieManager cm = new CookieManager(); - CookieHandler.setDefault(cm); WebEngine webEngine = webView.getEngine(); webEngine.setJavaScriptEnabled(true); webEngine.titleProperty().addListener((o, ov, nv) -> { if (nv != null) { logger.debug("YouTubeSignIn [loading-page={}]", nv); if (nv.contains("code=")) { - configData.refreshAccounts(); + //configData.refreshAccounts(); - String code = Stream.of(nv.split("&")) + final String authorizationCode = Stream.of(nv.split("&")) .filter(query -> query.startsWith("Success code=")) - .collect(Collectors.joining()).substring(13); - - logger.debug("YouTubeSignIn [returned-code={}]", code); - try { - OAuth2Tokens tokens = oauth2.getAccessTokens(code); - oauth2.setTokens(tokens); + .collect(Collectors.joining()) + .substring(13); - YouTubeAccount account = new YouTubeAccount(tokens); + logger.debug("YouTubeSignIn [returned-code={}]", authorizationCode); - configData.addAccount(account); + try { + oAuth2Manager.addAccount(authorizationCode); btnExitSignIn.fire(); } catch (Exception e) { @@ -124,45 +126,31 @@ public void initialize(URL location, ResourceBundle resources) { }); webViewLoading.visibleProperty().bind(webEngine.getLoadWorker().stateProperty().isEqualTo(Worker.State.SUCCEEDED).not()); - configData.accountListChangedProperty().addListener((lcl) -> { - config.save(); - - List listItems = configData.getAccounts() - .stream() - .filter(Objects::nonNull) - .filter(account -> account.getChannelId() != null && account.getThumbUrl() != null - && account.getUsername() != null) - .map(SettingsAccountItemView::new) - .collect(Collectors.toList()); - - runLater(() -> { - accountList.getItems().clear(); - accountList.getItems().addAll(listItems); - }); - }); - - configData.triggerAccountListChanged(); + reloadAccountList(); btnSave.setOnAction(ae -> runLater(() -> btnClose.fire())); closeIcon.setImage(ImageLoader.CLOSE.getImage()); btnClose.setOnAction(ae -> runLater(() -> { logger.debug("Saving Settings"); + ConfigData data = config.getDataObject(); - data.setAutoLoadStats(autoLoadStats.isSelected()); - data.setPrefixReplies(prefixReply.isSelected()); data.setArchiveThumbs(downloadThumbs.isSelected()); + data.setAutoLoadStats(autoLoadStats.isSelected()); data.setCustomApiKey(customKey.isSelected()); - data.setYoutubeApiKey(youtubeApiKey.getText()); data.setFilterDuplicatesOnCopy(filterDuplicatesOnCopy.isSelected()); + data.setGrabHeldForReview(grabHeldForReview.isSelected()); + data.setGrabLikelySpam(grabLikelySpam.isSelected()); + data.setPrefixReplies(prefixReply.isSelected()); + data.setYoutubeApiKey(youtubeApiKey.getText()); config.setDataObject(data); config.save(); if (customKey.isSelected()) { - FXMLSuite.setYouTubeApiKey(data.getYoutubeApiKey()); + CommentSuite.setYouTubeApiKey(data.getYoutubeApiKey()); } else { - FXMLSuite.setYouTubeApiKey(data.getDefaultApiKey()); + CommentSuite.setYouTubeApiKey(ConfigData.DEFAULT_API_KEY); } logger.debug("Closing Settings"); @@ -178,7 +166,7 @@ public void initialize(URL location, ResourceBundle resources) { vboxSignIn.setManaged(true); vboxSignIn.setVisible(true); vboxSettings.setDisable(true); - webView.getEngine().load(oauth2.getAuthUrl()); + webView.getEngine().load(OAuth2Manager.WEB_LOGIN_URL); })); btnExitSignIn.setOnAction(ae -> runLater(() -> { @@ -243,6 +231,33 @@ public void initialize(URL location, ResourceBundle resources) { github.setOnAction(ae -> browserUtil.open("https://github.com/mattwright324/youtube-comment-suite")); } + private void reloadAccountList() { + config.save(); + + List listItems = configData.getAccounts() + .stream() + .filter(Objects::nonNull) + .filter(account -> account.getChannelId() != null && account.getThumbUrl() != null + && account.getUsername() != null) + .map(SettingsAccountItemView::new) + .collect(Collectors.toList()); + + runLater(() -> { + accountList.getItems().clear(); + accountList.getItems().addAll(listItems); + }); + } + + @Subscribe + public void accountAddEvent(final AccountAddEvent accountAddEvent) { + reloadAccountList(); + } + + @Subscribe + public void accountDeleteEvent(final AccountDeleteEvent accountDeleteEvent) { + reloadAccountList(); + } + private void deleteDirectoryContents(String dir) { File file = new File(dir); diff --git a/src/main/java/io/mattw/youtube/commentsuite/fxml/SettingsAccountItemView.java b/src/main/java/io/mattw/youtube/commentsuite/fxml/SettingsAccountItemView.java index c60b34f..ed84a86 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/fxml/SettingsAccountItemView.java +++ b/src/main/java/io/mattw/youtube/commentsuite/fxml/SettingsAccountItemView.java @@ -1,6 +1,7 @@ package io.mattw.youtube.commentsuite.fxml; import io.mattw.youtube.commentsuite.*; +import io.mattw.youtube.commentsuite.oauth2.YouTubeAccount; import javafx.fxml.FXML; import javafx.fxml.FXMLLoader; import javafx.scene.control.Button; @@ -12,9 +13,6 @@ import java.io.IOException; -/** - * @author mattwright324 - */ public class SettingsAccountItemView extends HBox implements ImageCache { private static final Logger logger = LogManager.getLogger(); @@ -23,15 +21,15 @@ public class SettingsAccountItemView extends HBox implements ImageCache { @FXML private Label accountName; @FXML private Button btnRemove; - private ConfigFile configFile; - private ConfigData configData; + private final ConfigFile configFile; + private final ConfigData configData; - private YouTubeAccount account; + private final YouTubeAccount account; - public SettingsAccountItemView(YouTubeAccount account) { + public SettingsAccountItemView(final YouTubeAccount account) { this.account = account; - configFile = FXMLSuite.getConfig(); + configFile = CommentSuite.getConfig(); configData = configFile.getDataObject(); FXMLLoader loader = new FXMLLoader(getClass().getResource("SettingsAccountItemView.fxml")); diff --git a/src/main/java/io/mattw/youtube/commentsuite/YouTubeSearchItem.java b/src/main/java/io/mattw/youtube/commentsuite/fxml/YouTubeSearchItem.java similarity index 85% rename from src/main/java/io/mattw/youtube/commentsuite/YouTubeSearchItem.java rename to src/main/java/io/mattw/youtube/commentsuite/fxml/YouTubeSearchItem.java index 7c5805d..5cd18dc 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/YouTubeSearchItem.java +++ b/src/main/java/io/mattw/youtube/commentsuite/fxml/YouTubeSearchItem.java @@ -1,11 +1,8 @@ -package io.mattw.youtube.commentsuite; +package io.mattw.youtube.commentsuite.fxml; import com.google.api.services.youtube.model.SearchResult; import io.mattw.youtube.commentsuite.db.YouTubeObject; -/** - * @author mattwright324 - */ public class YouTubeSearchItem extends YouTubeObject { public YouTubeSearchItem(SearchResult searchResult) { diff --git a/src/main/java/io/mattw/youtube/commentsuite/oauth2/OAuth2Manager.java b/src/main/java/io/mattw/youtube/commentsuite/oauth2/OAuth2Manager.java new file mode 100644 index 0000000..3093ef8 --- /dev/null +++ b/src/main/java/io/mattw/youtube/commentsuite/oauth2/OAuth2Manager.java @@ -0,0 +1,157 @@ +package io.mattw.youtube.commentsuite.oauth2; + +import com.google.api.client.googleapis.json.GoogleJsonResponseException; +import com.google.api.services.youtube.YouTube; +import com.google.api.services.youtube.model.*; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.mattw.youtube.commentsuite.CommentSuite; +import io.mattw.youtube.commentsuite.ConfigData; +import io.mattw.youtube.commentsuite.ConfigFile; +import io.mattw.youtube.commentsuite.db.CommentDatabase; +import io.mattw.youtube.commentsuite.db.YouTubeChannel; +import io.mattw.youtube.commentsuite.util.UTF8UrlEncoder; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jsoup.Connection; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; + +import java.io.IOException; +import java.lang.reflect.Modifier; +import java.sql.SQLException; +import java.util.Collections; +import java.util.Optional; + +public class OAuth2Manager { + + private static final Logger logger = LogManager.getLogger(); + private static final Gson gson = new GsonBuilder().excludeFieldsWithModifiers(Modifier.FINAL).create(); + + private static final String OAUTH2_TOKENS_URL = "https://accounts.google.com/o/oauth2/token"; + private static final String CLIENT_ID = "972416191049-htqcmg31u2t7hbd1ncen2e2jsg68cnqn.apps.googleusercontent.com"; + private static final String CLIENT_SECRET = "QuTdoA-KArupKMWwDrrxOcoS"; + private static final String REDIRECT_URI = "urn:ietf:wg:oauth:2.0:oob"; + + public static final String WEB_LOGIN_URL = String.format("https://accounts.google.com/o/oauth2/auth?client_id=%s&redirect_uri=%s&&response_type=code&&scope=%s", + CLIENT_ID, UTF8UrlEncoder.encode(REDIRECT_URI), UTF8UrlEncoder.encode("https://www.googleapis.com/auth/youtube.force-ssl")); + + private final YouTube youTube; + private final ConfigFile configFile; + private final ConfigData configData; + private final CommentDatabase database; + + public OAuth2Manager() { + this.youTube = CommentSuite.getYouTube(); + this.configFile = CommentSuite.getConfig(); + this.configData = configFile.getDataObject(); + this.database = CommentSuite.getDatabase(); + } + + /** + * Create account tokens and details from login authorization code retrieved from {@link #WEB_LOGIN_URL} + */ + public YouTubeAccount addAccount(final String authorizationCode) throws IOException { + final OAuth2Tokens tokens = getOAuthTokens(authorizationCode); + final YouTubeAccount account = new YouTubeAccount(tokens); + + getAndSetAccountDetails(account); + + configData.addAccount(account); + configFile.save(); + + return account; + } + + /** + * Get a new access token on the account because the current one has expired. + */ + public YouTubeAccount getNewAccessToken(final YouTubeAccount account) throws IOException { + final OAuth2Tokens newTokens = getNewAccessToken(account.getTokens()); + + account.setTokens(newTokens); + configData.getAccount(account.getChannelId()).setTokens(newTokens); + configFile.save(); + + return account; + } + + private Connection getOauth2Connection() { + return Jsoup.connect(OAUTH2_TOKENS_URL) + .ignoreContentType(true) + .data("client_id", CLIENT_ID) + .data("client_secret", CLIENT_SECRET); + } + + private OAuth2Tokens getOAuthTokens(final String authorizationCode) throws IOException { + logger.debug("Grabbing OAuth2Tokens from authorization code"); + final Document result = getOauth2Connection() + .data("code", authorizationCode) + .data("redirect_uri", REDIRECT_URI) + .data("grant_type", "authorization_code") + .post(); + + return gson.fromJson(result.text(), OAuth2Tokens.class); + } + + private OAuth2Tokens getNewAccessToken(final OAuth2Tokens oldTokens) throws IOException { + logger.debug("Refreshing OAuth2 Access Token that has expired {}", oldTokens); + final Document result = getOauth2Connection() + .data("refresh_token", oldTokens.getRefreshToken()) + .data("grant_type", "refresh_token") + .post(); + + final OAuth2Tokens newTokens = gson.fromJson(result.text(), OAuth2Tokens.class); + newTokens.setRefreshToken(oldTokens.getRefreshToken()); + return newTokens; + } + + private void getAndSetAccountDetails(final YouTubeAccount account) throws IOException { + logger.debug("Grabbing account details"); + + try { + final ChannelListResponse response = youTube.channels() + .list("snippet") + .setOauthToken(account.getTokens().getAccessToken()) + .setMine(true) + .execute(); + + if (response.getItems() == null || response.getItems().isEmpty()) { + throw new IOException("Response had no channel"); + } + + final Channel channelItem = response.getItems().get(0); + final ChannelSnippet snippet = channelItem.getSnippet(); + + if (snippet == null) { + throw new IOException("Channel had no snippet"); + } + + account.setChannelId(channelItem.getId()); + account.setUsername(snippet.getTitle()); + account.setThumbUrl(Optional.ofNullable(snippet.getThumbnails()) + .map(ThumbnailDetails::getMedium) + .map(Thumbnail::getUrl) + .orElse("https://placehold.it/128x128")); + + try { + final YouTubeChannel channel = new YouTubeChannel(channelItem); + database.insertChannels(Collections.singletonList(channel)); + database.commit(); + } catch (SQLException e) { + logger.error("Unable to insert account channel into database.", e); + } + } catch (GoogleJsonResponseException e) { + if (e.getStatusCode() == 401) { + logger.warn("Tokens have expired for account [username={}]", account.getUsername()); + } else { + logger.error("An unexpected error occurred."); + throw e; + } + } catch (IOException e) { + logger.error("Failed to query for account channel info."); + throw e; + } + } + +} diff --git a/src/main/java/io/mattw/youtube/commentsuite/oauth2/OAuth2Tokens.java b/src/main/java/io/mattw/youtube/commentsuite/oauth2/OAuth2Tokens.java new file mode 100644 index 0000000..803dff3 --- /dev/null +++ b/src/main/java/io/mattw/youtube/commentsuite/oauth2/OAuth2Tokens.java @@ -0,0 +1,50 @@ +package io.mattw.youtube.commentsuite.oauth2; + +import org.apache.commons.lang3.builder.ReflectionToStringBuilder; + +import java.io.Serializable; + +public class OAuth2Tokens implements Serializable { + + private String access_token; + private String token_type; + private String refresh_token; + private int expires_in; + + public String getAccessToken() { + return access_token; + } + + public void setAccessToken(String access_token) { + this.access_token = access_token; + } + + public String getTokenType() { + return token_type; + } + + public void setTokenType(String token_type) { + this.token_type = token_type; + } + + public String getRefreshToken() { + return refresh_token; + } + + public void setRefreshToken(String refresh_token) { + this.refresh_token = refresh_token; + } + + public int getExpiresIn() { + return expires_in; + } + + public void setExpiresIn(int expires_in) { + this.expires_in = expires_in; + } + + public String toString() { + return ReflectionToStringBuilder.toString(this); + } + +} diff --git a/src/main/java/io/mattw/youtube/commentsuite/oauth2/YouTubeAccount.java b/src/main/java/io/mattw/youtube/commentsuite/oauth2/YouTubeAccount.java new file mode 100644 index 0000000..703d3b1 --- /dev/null +++ b/src/main/java/io/mattw/youtube/commentsuite/oauth2/YouTubeAccount.java @@ -0,0 +1,56 @@ +package io.mattw.youtube.commentsuite.oauth2; + +import java.io.Serializable; + +public class YouTubeAccount implements Serializable { + + private String username; + private String channelId; + private String thumbUrl; + private OAuth2Tokens tokens; + + public YouTubeAccount(OAuth2Tokens tokens) { + this.tokens = tokens; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public String getChannelId() { + return channelId; + } + + public void setChannelId(String channelId) { + this.channelId = channelId; + } + + public String getThumbUrl() { + return thumbUrl; + } + + public void setThumbUrl(String thumbUrl) { + this.thumbUrl = thumbUrl; + } + + public OAuth2Tokens getTokens() { + return tokens; + } + + public void setTokens(OAuth2Tokens tokens) { + this.tokens = tokens; + } + + public String toString() { + return username; + } + + public boolean equals(Object o) { + return o instanceof YouTubeAccount && ((YouTubeAccount) o).getChannelId() != null && ((YouTubeAccount) o).getChannelId().equals(channelId); + } + +} diff --git a/src/main/java/io/mattw/youtube/commentsuite/refresh/ChannelConsumer.java b/src/main/java/io/mattw/youtube/commentsuite/refresh/ChannelConsumer.java index 6b2e13f..8e9d5d2 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/refresh/ChannelConsumer.java +++ b/src/main/java/io/mattw/youtube/commentsuite/refresh/ChannelConsumer.java @@ -1,6 +1,6 @@ package io.mattw.youtube.commentsuite.refresh; -import io.mattw.youtube.commentsuite.FXMLSuite; +import io.mattw.youtube.commentsuite.CommentSuite; import io.mattw.youtube.commentsuite.db.CommentDatabase; import io.mattw.youtube.commentsuite.db.YouTubeChannel; import io.mattw.youtube.commentsuite.util.ElapsedTime; @@ -27,7 +27,7 @@ public class ChannelConsumer extends ConsumerMultiProducer { private final Set concurrentNewChannelSet = ConcurrentHashMap.newKeySet(); public ChannelConsumer() { - this.database = FXMLSuite.getDatabase(); + this.database = CommentSuite.getDatabase(); } @Override diff --git a/src/main/java/io/mattw/youtube/commentsuite/refresh/ChannelProducer.java b/src/main/java/io/mattw/youtube/commentsuite/refresh/ChannelProducer.java index 79713f7..5020044 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/refresh/ChannelProducer.java +++ b/src/main/java/io/mattw/youtube/commentsuite/refresh/ChannelProducer.java @@ -3,7 +3,7 @@ import com.google.api.client.googleapis.json.GoogleJsonResponseException; import com.google.api.services.youtube.YouTube; import com.google.api.services.youtube.model.ChannelListResponse; -import io.mattw.youtube.commentsuite.FXMLSuite; +import io.mattw.youtube.commentsuite.CommentSuite; import io.mattw.youtube.commentsuite.db.YouTubeChannel; import io.mattw.youtube.commentsuite.util.ElapsedTime; import io.mattw.youtube.commentsuite.util.ExecutorGroup; @@ -30,7 +30,7 @@ public class ChannelProducer extends ConsumerMultiProducer { private final YouTube youTube; public ChannelProducer() { - this.youTube = FXMLSuite.getYouTube(); + this.youTube = CommentSuite.getYouTube(); } @Override @@ -82,7 +82,7 @@ private void produceChannels(final List channelIds) { try { final ChannelListResponse cl = youTube.channels() .list("snippet") - .setKey(FXMLSuite.getYouTubeApiKey()) + .setKey(CommentSuite.getYouTubeApiKey()) .setId(String.join(",", channelIds)) .setMaxResults(50L) .execute(); diff --git a/src/main/java/io/mattw/youtube/commentsuite/refresh/CommentConsumer.java b/src/main/java/io/mattw/youtube/commentsuite/refresh/CommentConsumer.java index faad485..c2cd326 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/refresh/CommentConsumer.java +++ b/src/main/java/io/mattw/youtube/commentsuite/refresh/CommentConsumer.java @@ -1,6 +1,6 @@ package io.mattw.youtube.commentsuite.refresh; -import io.mattw.youtube.commentsuite.FXMLSuite; +import io.mattw.youtube.commentsuite.CommentSuite; import io.mattw.youtube.commentsuite.db.CommentDatabase; import io.mattw.youtube.commentsuite.db.YouTubeComment; import io.mattw.youtube.commentsuite.util.ElapsedTime; @@ -20,13 +20,15 @@ public class CommentConsumer extends ConsumerMultiProducer { private final ExecutorGroup executorGroup = new ExecutorGroup(10); + private final boolean moderated; private final CommentDatabase database; private AtomicLong totalComments = new AtomicLong(); private AtomicLong newComments = new AtomicLong(); - public CommentConsumer() { - this.database = FXMLSuite.getDatabase(); + public CommentConsumer(final boolean moderated) { + this.moderated = moderated; + this.database = CommentSuite.getDatabase(); } @Override @@ -35,7 +37,7 @@ public void startProducing() { } private void produce() { - logger.debug("Starting CommentConsumer"); + logger.debug("Starting CommentConsumer " + moderated); final ElapsedTime elapsedTime = new ElapsedTime(); final List comments = new ArrayList<>(); @@ -61,20 +63,32 @@ private void produce() { insertComments(comments); } - logger.debug("Ending CommentConsumer {}", shouldKeepAlive()); + logger.debug("Ending CommentConsumer " + moderated); } private void insertComments(final List comments) { try { + comments.removeIf(comment -> moderated + && (comment.getModerationStatus() == null || + comment.getModerationStatus() == ModerationStatus.PUBLISHED)); + final List commentIds = comments.stream() .map(YouTubeComment::getId) .collect(Collectors.toList()); - newComments.addAndGet(database.countCommentsNotExisting(commentIds)); totalComments.addAndGet(comments.size()); - // Actually insert now - database.insertComments(comments); + if (moderated) { + newComments.addAndGet(database.countModeratedCommentsNotExisting(commentIds)); + + comments.removeIf(comment -> comment.getModerationStatus() == null || comment.getModerationStatus() == ModerationStatus.PUBLISHED); + + database.insertModeratedComments(comments); + } else { + newComments.addAndGet(database.countCommentsNotExisting(commentIds)); + database.insertComments(comments); + } + comments.clear(); } catch (SQLException e) { logger.error("Error on comment submit", e); diff --git a/src/main/java/io/mattw/youtube/commentsuite/refresh/CommentThreadProducer.java b/src/main/java/io/mattw/youtube/commentsuite/refresh/CommentThreadProducer.java index 0882e97..1c50dc5 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/refresh/CommentThreadProducer.java +++ b/src/main/java/io/mattw/youtube/commentsuite/refresh/CommentThreadProducer.java @@ -5,10 +5,14 @@ import com.google.api.services.youtube.YouTube; import com.google.api.services.youtube.model.CommentThread; import com.google.api.services.youtube.model.CommentThreadListResponse; -import io.mattw.youtube.commentsuite.FXMLSuite; +import com.google.api.services.youtube.model.CommentThreadReplies; +import io.mattw.youtube.commentsuite.*; import io.mattw.youtube.commentsuite.db.CommentDatabase; import io.mattw.youtube.commentsuite.db.YouTubeComment; import io.mattw.youtube.commentsuite.db.YouTubeVideo; +import io.mattw.youtube.commentsuite.oauth2.OAuth2Manager; +import io.mattw.youtube.commentsuite.oauth2.OAuth2Tokens; +import io.mattw.youtube.commentsuite.oauth2.YouTubeAccount; import io.mattw.youtube.commentsuite.util.ExecutorGroup; import io.mattw.youtube.commentsuite.util.StringTuple; import org.apache.commons.lang3.StringUtils; @@ -18,8 +22,11 @@ import java.sql.SQLException; import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; +import static io.mattw.youtube.commentsuite.refresh.ModerationStatus.PUBLISHED; + public class CommentThreadProducer extends ConsumerMultiProducer { private static final Logger logger = LogManager.getLogger(); @@ -27,13 +34,23 @@ public class CommentThreadProducer extends ConsumerMultiProducer { private final ExecutorGroup executorGroup = new ExecutorGroup(10); private final RefreshOptions options; + private final RefreshCommentPages pages; private final YouTube youTube; private final CommentDatabase database; + private final ModerationStatus moderationStatus; + private final ConfigData configData = CommentSuite.getConfig().getDataObject(); + private final OAuth2Manager oAuth2Manager = CommentSuite.getOauth2Manager(); + + public CommentThreadProducer(final RefreshOptions options, final RefreshCommentPages pages) { + this(options, pages, PUBLISHED); + } - public CommentThreadProducer(final RefreshOptions options) { + public CommentThreadProducer(final RefreshOptions options, final RefreshCommentPages pages, final ModerationStatus moderationStatus) { this.options = options; - this.youTube = FXMLSuite.getYouTube(); - this.database = FXMLSuite.getDatabase(); + this.pages = pages; + this.moderationStatus = moderationStatus; + this.youTube = CommentSuite.getYouTube(); + this.database = CommentSuite.getDatabase(); } @Override @@ -41,8 +58,38 @@ public void startProducing() { executorGroup.submitAndShutdown(this::produce); } + private String getOauthToken(String channelId) { + if (moderationStatus == PUBLISHED) { + return null; + } + + final YouTubeAccount account = configData.getAccount(channelId); + final String accessToken = Optional.ofNullable(account) + .map(YouTubeAccount::getTokens) + .map(OAuth2Tokens::getAccessToken) + .orElse(null); + + logger.debug(accessToken); + + return accessToken; + } + + private void refreshOauth2(final String channelId) { + if (moderationStatus == PUBLISHED) { + return; + } + + logger.debug("Refreshing OAuth2 Tokens for {}", channelId); + + try { + oAuth2Manager.getNewAccessToken(configData.getAccount(channelId)); + } catch (Exception e) { + e.printStackTrace(); + } + } + private void produce() { - logger.debug("Starting CommentThreadProducer"); + logger.debug("Starting CommentThreadProducer " + moderationStatus); while (shouldKeepAlive()) { final YouTubeVideo video = getBlockingQueue().poll(); @@ -51,29 +98,40 @@ private void produce() { continue; } + if (moderationStatus != PUBLISHED && !configData.isSignedIn(video.getChannelId())) { + logger.warn("Authorization required for {} commentThreads on {} but not signed in", moderationStatus, video.getId()); + awaitMillis(100); + continue; + } + send(video.getChannelId()); - int attempts = 0; + int attempts = 1; CommentThreadListResponse response; String pageToken = ""; int page = 1; - RefreshCommentPages commentPages = options.getCommentPages(); final int maxAttempts = 5; do { try { - logger.info("{} - {}", video.getId(), video.getTitle()); + logger.info("{} - {} {}", video.getId(), moderationStatus, video.getTitle()); do { + logger.debug("{} {}", moderationStatus, attempts); + response = youTube.commentThreads() - .list("snippet") - .setKey(FXMLSuite.getYouTubeApiKey()) + .list(moderationStatus.getPart()) + .setKey(CommentSuite.getYouTubeApiKey()) + .setOauthToken(getOauthToken(video.getChannelId())) .setVideoId(video.getId()) .setMaxResults(100L) .setOrder(options.getCommentOrder().name()) .setPageToken(pageToken) + .setModerationStatus(moderationStatus.getApiValue()) .execute(); pageToken = response.getNextPageToken(); + logger.debug(response); + try { // Maybe comments were re-enabled if we got a 403 in the past. database.updateVideoHttpCode(video.getId(), 200); @@ -92,6 +150,19 @@ private void produce() { .collect(Collectors.toList()); sendCollection(comments, YouTubeComment.class); + if (moderationStatus != PUBLISHED) { + final List replies = items.stream() + .map(CommentThread::getReplies) + .map(CommentThreadReplies::getComments) + .flatMap(List::stream) + .map(comment -> new YouTubeComment(comment, video.getId())) + .peek(comment -> comment.setModerationStatus(moderationStatus)) + .filter(comment -> StringUtils.isNotEmpty(comment.getChannelId())) + .collect(Collectors.toList()); + + sendCollection(replies, YouTubeComment.class); + } + final List channelIds = comments.stream() .map(YouTubeComment::getChannelId) .distinct() @@ -106,13 +177,13 @@ private void produce() { */ final List replyThreads = comments.stream() .filter(comment -> comment.getReplyCount() > 0) - .filter(comment -> options.getReplyPages() != RefreshReplyPages.NONE) + .filter(comment -> options.getReplyPages() != RefreshCommentPages.NONE) .map(comment -> new StringTuple(comment.getId(), video.getId())) .collect(Collectors.toList()); sendCollection(replyThreads, StringTuple.class); awaitMillis(50); - } while (pageToken != null && page++ < commentPages.getPageCount() && !isHardShutdown()); + } while (pageToken != null && page++ < pages.getPageCount() && !isHardShutdown()); break; } catch (Exception e) { @@ -137,6 +208,19 @@ private void produce() { sendMessage(Level.ERROR, e, message); + attempts++; + } else if (ge.getStatusCode() == 401 && moderationStatus != PUBLISHED) { + final String message = String.format("[%s/%s] Authorization failed [videoId=%s]", + attempts, + maxAttempts, + video.getId()); + + sendMessage(Level.WARN, message); + sendMessage(Level.WARN, "Trying to refresh Oauth2 access token"); + + refreshOauth2(video.getChannelId()); + CommentSuite.getConfig().save(); + attempts++; } else if (ge.getStatusCode() == 403) { final String message = String.format("Comments Disabled [videoId=%s]", video.getId()); @@ -165,13 +249,13 @@ private void produce() { attempts++; } } - } while (attempts < maxAttempts && !isHardShutdown()); + } while (attempts <= maxAttempts && !isHardShutdown()); addProcessed(1); awaitMillis(500); } - logger.debug("Ending CommentThreadProducer"); + logger.debug("Ending CommentThreadProducer " + moderationStatus); } @Override diff --git a/src/main/java/io/mattw/youtube/commentsuite/refresh/ConsumerMultiProducer.java b/src/main/java/io/mattw/youtube/commentsuite/refresh/ConsumerMultiProducer.java index 5e5951b..2cd0c9f 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/refresh/ConsumerMultiProducer.java +++ b/src/main/java/io/mattw/youtube/commentsuite/refresh/ConsumerMultiProducer.java @@ -26,6 +26,7 @@ public abstract class ConsumerMultiProducer { private List> keepAliveWith = new ArrayList<>(); private Map, List>> consumersByClass = new HashMap<>(); + private Map>> consumersByKey = new HashMap<>(); private BlockingQueue blockingQueue = new LinkedBlockingQueue<>(); private boolean startProduceOnFirstAccept = false; private boolean didProduceOnFirstAccept = false; @@ -52,6 +53,23 @@ public

void produceTo(ConsumerMultiProducer

consumer, Class

clazz) { }); } + /** + * @param consumer consuemr + * @param clazz needed for sendCollection() + * @param

consumer must consume type of clazz + */ + public

void produceTo(ConsumerMultiProducer

consumer, Class

clazz, String key) { + consumersByKey.computeIfPresent(key, (key1, value) -> { + value.add(consumer); + return value; + }); + consumersByKey.computeIfAbsent(key, (key1) -> { + List> list = new ArrayList<>(); + list.add(consumer); + return list; + }); + } + /** * Start producing using the ExecutorGroup */ @@ -134,6 +152,24 @@ public

void send(P object) { } } + public

void sendCollection(Collection

objects, Class

clazz, String key) { + if (!consumersByKey.containsKey(key) || objects.isEmpty()) { + return; + } + for (ConsumerMultiProducer consumer : consumersByKey.get(key)) { + ((ConsumerMultiProducer

) consumer).accept(objects); + } + } + + public

void send(P object, String key) { + if (!consumersByKey.containsKey(key)) { + return; + } + for (ConsumerMultiProducer consumer : consumersByKey.get(key)) { + ((ConsumerMultiProducer

) consumer).accept(object); + } + } + public void setMessageFunc(TriConsumer messageFunc) { this.messageFunc = messageFunc; } diff --git a/src/main/java/io/mattw/youtube/commentsuite/refresh/GroupRefresh.java b/src/main/java/io/mattw/youtube/commentsuite/refresh/GroupRefresh.java index ab3ab35..1d016ac 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/refresh/GroupRefresh.java +++ b/src/main/java/io/mattw/youtube/commentsuite/refresh/GroupRefresh.java @@ -2,10 +2,11 @@ import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.googleapis.json.GoogleJsonResponseException; -import io.mattw.youtube.commentsuite.FXMLSuite; +import io.mattw.youtube.commentsuite.CommentSuite; import io.mattw.youtube.commentsuite.db.*; import io.mattw.youtube.commentsuite.util.ElapsedTime; import io.mattw.youtube.commentsuite.util.StringTuple; +import io.mattw.youtube.commentsuite.util.Threads; import javafx.beans.property.*; import javafx.collections.FXCollections; import javafx.collections.ObservableList; @@ -21,8 +22,9 @@ import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.stream.Stream; +import static io.mattw.youtube.commentsuite.refresh.ModerationStatus.HELD_FOR_REVIEW; +import static io.mattw.youtube.commentsuite.refresh.ModerationStatus.LIKELY_SPAM; import static javafx.application.Platform.runLater; public class GroupRefresh extends Thread implements RefreshInterface { @@ -40,6 +42,8 @@ public class GroupRefresh extends Thread implements RefreshInterface { private final LongProperty totalVideosProperty = new SimpleLongProperty(0); private final LongProperty newCommentsProperty = new SimpleLongProperty(0); private final LongProperty totalCommentsProperty = new SimpleLongProperty(0); + private final LongProperty newModeratedProperty = new SimpleLongProperty(0); + private final LongProperty totalModeratedProperty = new SimpleLongProperty(0); private final LongProperty newViewersProperty = new SimpleLongProperty(0); private final LongProperty totalViewersProperty = new SimpleLongProperty(0); @@ -50,10 +54,10 @@ public class GroupRefresh extends Thread implements RefreshInterface { private final VideoIdProducer videoIdProducer; private final UniqueVideoIdProducer uniqueVideoIdProducer; private final VideoProducer videoProducer; - private final CommentThreadProducer commentThreadProducer; + private final CommentThreadProducer commentThreadProducer, reviewThreadProducer, spamThreadProducer; private final ReplyProducer replyProducer; private final ChannelProducer channelProducer; - private final CommentConsumer commentConsumer; + private final CommentConsumer commentConsumer, moderatedCommentConsumer; private final ChannelConsumer channelConsumer; private boolean hardShutdown = false; @@ -62,15 +66,18 @@ public class GroupRefresh extends Thread implements RefreshInterface { public GroupRefresh(Group group, RefreshOptions options) { this.group = group; this.options = options; - this.database = FXMLSuite.getDatabase(); + this.database = CommentSuite.getDatabase(); this.videoIdProducer = new VideoIdProducer(); this.uniqueVideoIdProducer = new UniqueVideoIdProducer(); this.videoProducer = new VideoProducer(options); - this.commentThreadProducer = new CommentThreadProducer(options); + this.commentThreadProducer = new CommentThreadProducer(options, options.getCommentPages()); + this.reviewThreadProducer = new CommentThreadProducer(options, options.getReviewPages(), HELD_FOR_REVIEW); + this.spamThreadProducer = new CommentThreadProducer(options, options.getReplyPages(), LIKELY_SPAM); this.replyProducer = new ReplyProducer(options); this.channelProducer = new ChannelProducer(); - this.commentConsumer = new CommentConsumer(); + this.commentConsumer = new CommentConsumer(false); + this.moderatedCommentConsumer = new CommentConsumer(true); this.channelConsumer = new ChannelConsumer(); } @@ -88,31 +95,26 @@ public void run() { uniqueVideoIdProducer.setMessageFunc(this::postMessage); videoProducer.produceTo(commentThreadProducer, YouTubeVideo.class); + videoProducer.produceTo(reviewThreadProducer, YouTubeVideo.class, HELD_FOR_REVIEW.name()); + videoProducer.produceTo(spamThreadProducer, YouTubeVideo.class, LIKELY_SPAM.name()); videoProducer.setMessageFunc(this::postMessage); - try { - // Parse GroupItems to VideoIds - runLater(() -> statusStepProperty.setValue("Grabbing Videos")); - videoIdProducer.startProducing(); - await(videoIdProducer, "Await videoIdProducer over"); - database.commit(); - - uniqueVideoIdProducer.startProducing(); - await(uniqueVideoIdProducer, "Await uniqueVideoIdProducer over"); - - // Parse VideoIds to Videos - runLater(() -> statusStepProperty.setValue("Grabbing Video Data")); - videoProducer.startProducing(); - await(videoProducer, "Await videoProducer over"); - } catch (SQLException | InterruptedException e) { - postMessage(Level.FATAL, e, null); - } - commentThreadProducer.produceTo(commentConsumer, YouTubeComment.class); commentThreadProducer.produceTo(channelProducer, String.class); commentThreadProducer.produceTo(replyProducer, StringTuple.class); commentThreadProducer.setMessageFunc(this::postMessage); + reviewThreadProducer.produceTo(moderatedCommentConsumer, YouTubeComment.class); + reviewThreadProducer.produceTo(channelProducer, String.class); + //reviewThreadProducer.produceTo(replyProducer, StringTuple.class); + reviewThreadProducer.setMessageFunc(this::postMessage); + + spamThreadProducer.produceTo(moderatedCommentConsumer, YouTubeComment.class); + spamThreadProducer.produceTo(channelProducer, String.class); + //spamThreadProducer.produceTo(replyProducer, StringTuple.class); + spamThreadProducer.setMessageFunc(this::postMessage); + spamThreadProducer.setStartProduceOnFirstAccept(true); + replyProducer.produceTo(commentConsumer, YouTubeComment.class); replyProducer.produceTo(channelConsumer, YouTubeChannel.class); replyProducer.keepAliveWith(commentThreadProducer); @@ -128,33 +130,60 @@ public void run() { commentConsumer.setStartProduceOnFirstAccept(true); commentConsumer.setMessageFunc(this::postMessage); + moderatedCommentConsumer.keepAliveWith(reviewThreadProducer, spamThreadProducer); + moderatedCommentConsumer.setStartProduceOnFirstAccept(true); + moderatedCommentConsumer.setMessageFunc(this::postMessage); + channelConsumer.keepAliveWith(commentThreadProducer, channelProducer, replyProducer); channelConsumer.setStartProduceOnFirstAccept(true); channelConsumer.setMessageFunc(this::postMessage); try { + // Parse GroupItems to VideoIds + runLater(() -> statusStepProperty.setValue("Grabbing Videos")); + videoIdProducer.startProducing(); + await(videoIdProducer, "Await videoIdProducer over"); + database.commit(); + + uniqueVideoIdProducer.startProducing(); + await(uniqueVideoIdProducer, "Await uniqueVideoIdProducer over"); + + // Parse VideoIds to Videos + runLater(() -> statusStepProperty.setValue("Grabbing Video Data")); + videoProducer.startProducing(); + await(videoProducer, "Await videoProducer over"); + // Parse Videos to CommentThreads, Replies, Channels runLater(() -> statusStepProperty.setValue("Grabbing Comments")); startProgressThread(); commentThreadProducer.startProducing(); + if (!reviewThreadProducer.getBlockingQueue().isEmpty()) { + reviewThreadProducer.startProducing(); + } + + if (!spamThreadProducer.getBlockingQueue().isEmpty()) { + spamThreadProducer.startProducing(); + } + await(commentThreadProducer, "Await commentThreadProducer over"); await(replyProducer, "Await replyProducer over"); await(channelProducer, "Await channelProducer over"); await(commentConsumer, "Await commentConsumer over"); await(channelConsumer, "Await channelConsumer over"); - } catch (InterruptedException e) { - postMessage(Level.FATAL, e, null); - } - try { - database.commit(); + try { + database.commit(); - awaitMillis(100); + Threads.awaitMillis(100); - runLater(() -> statusStepProperty.setValue("Done")); - } catch (SQLException e) { - postMessage(Level.FATAL, e, "Failed to commit"); + runLater(() -> statusStepProperty.setValue("Done")); + } catch (SQLException e) { + postMessage(Level.FATAL, e, "Failed to commit"); + } + + } catch (SQLException | InterruptedException e) { + postMessage(Level.FATAL, e, null); } runLater(() -> { @@ -184,6 +213,7 @@ private void postMessage(final Level level, final Throwable error, final String final String time = formatter.format(LocalDateTime.now()); if (level == Level.FATAL) { + runLater(() -> statusStepProperty.setValue("[FATAL] " + statusStepProperty.getValue())); endedOnError = true; hardShutdown(); } @@ -229,7 +259,7 @@ private void startProgressThread() { final double percentage = totalProcessed / totalAccepted; runLater(() -> progressProperty.setValue(percentage)); - awaitMillis(100); + Threads.awaitMillis(100); } logger.debug("Ended Progress Thread"); }); @@ -243,6 +273,9 @@ private void pollProcessed() { newCommentsProperty.setValue(commentConsumer.getNewComments()); totalCommentsProperty.setValue(commentConsumer.getTotalComments()); + newModeratedProperty.setValue(moderatedCommentConsumer.getNewComments()); + totalModeratedProperty.setValue(moderatedCommentConsumer.getTotalComments()); + newViewersProperty.setValue(channelConsumer.getNewChannels()); totalViewersProperty.setValue(channelConsumer.getTotalChannels()); } @@ -265,7 +298,7 @@ private void startElapsedTimer() { elapsedTimeProperty.setValue(String.format("%s", elapsedTimer.humanReadableFormat())); pollProcessed(); }); - awaitMillis(27); + Threads.awaitMillis(27); } logger.debug("Ended Elapsed Timer"); @@ -287,18 +320,6 @@ private void startElapsedTimer() { */ } - private void awaitMillis(long millis) { - try { - Thread.sleep(millis); - } catch (InterruptedException ignored) { - } - } - - @Override - public void appendError(String error) { - - } - @Override public void hardShutdown() { videoIdProducer.setHardShutdown(true); @@ -333,6 +354,16 @@ public LongProperty totalCommentsProperty() { return totalCommentsProperty; } + @Override + public LongProperty newModeratedProperty() { + return newModeratedProperty; + } + + @Override + public LongProperty totalModeratedProperty() { + return totalModeratedProperty; + } + @Override public LongProperty newViewersProperty() { return newViewersProperty; diff --git a/src/main/java/io/mattw/youtube/commentsuite/refresh/ModerationStatus.java b/src/main/java/io/mattw/youtube/commentsuite/refresh/ModerationStatus.java new file mode 100644 index 0000000..6850a1a --- /dev/null +++ b/src/main/java/io/mattw/youtube/commentsuite/refresh/ModerationStatus.java @@ -0,0 +1,52 @@ +package io.mattw.youtube.commentsuite.refresh; + +import org.apache.commons.lang3.EnumUtils; + +import java.util.Map; + +public enum ModerationStatus { + PUBLISHED("published", "snippet"), + HELD_FOR_REVIEW("heldForReview", "snippet,replies"), + LIKELY_SPAM("likelySpam", "snippet,replies"); + + private static final Map lookup = EnumUtils.getEnumMap(ModerationStatus.class); + + private final String apiValue; + private final String part; + + ModerationStatus(final String apiValue, final String part) { + this.apiValue = apiValue; + this.part = part; + } + + public String getApiValue() { + return apiValue; + } + + public String getPart() { + return part; + } + + public static ModerationStatus fromName(final String name) { + if (name == null) { + return null; + } + + return lookup.getOrDefault(name, null); + } + + public static ModerationStatus fromApiValue(final String apiValue) { + if (apiValue == null) { + return null; + } + + for (ModerationStatus status : ModerationStatus.values()) { + if (apiValue.equals(status.getApiValue())) { + return status; + } + } + + return null; + } + +} diff --git a/src/main/java/io/mattw/youtube/commentsuite/refresh/RefreshInterface.java b/src/main/java/io/mattw/youtube/commentsuite/refresh/RefreshInterface.java index 8cec9a4..6d0eb6b 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/refresh/RefreshInterface.java +++ b/src/main/java/io/mattw/youtube/commentsuite/refresh/RefreshInterface.java @@ -9,14 +9,11 @@ /** * Common interface for group refreshing. * - * @author mattwright324 */ public interface RefreshInterface { void run(); - void appendError(String error); - void hardShutdown(); boolean isAlive(); @@ -31,6 +28,10 @@ public interface RefreshInterface { LongProperty totalCommentsProperty(); + LongProperty newModeratedProperty(); + + LongProperty totalModeratedProperty(); + LongProperty newViewersProperty(); LongProperty totalViewersProperty(); diff --git a/src/main/java/io/mattw/youtube/commentsuite/refresh/RefreshOptions.java b/src/main/java/io/mattw/youtube/commentsuite/refresh/RefreshOptions.java index d0ac881..24c876d 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/refresh/RefreshOptions.java +++ b/src/main/java/io/mattw/youtube/commentsuite/refresh/RefreshOptions.java @@ -10,7 +10,9 @@ public class RefreshOptions { private RefreshTimeframe timeframe = style.getTimeframe(); private RefreshCommentPages commentPages = style.getCommentPages(); private RefreshCommentOrder commentOrder = style.getCommentOrder(); - private RefreshReplyPages replyPages = style.getReplyPages(); + private RefreshCommentPages replyPages = style.getReplyPages(); + private RefreshCommentPages reviewPages = RefreshCommentPages.ALL; + private RefreshCommentPages spamPages = RefreshCommentPages.ALL; public RefreshStyle getStyle() { return style; @@ -44,14 +46,30 @@ public void setCommentOrder(RefreshCommentOrder commentOrder) { this.commentOrder = commentOrder; } - public RefreshReplyPages getReplyPages() { + public RefreshCommentPages getReplyPages() { return replyPages; } - public void setReplyPages(RefreshReplyPages replyPages) { + public void setReplyPages(RefreshCommentPages replyPages) { this.replyPages = replyPages; } + public RefreshCommentPages getReviewPages() { + return reviewPages; + } + + public void setReviewPages(RefreshCommentPages reviewPages) { + this.reviewPages = reviewPages; + } + + public RefreshCommentPages getSpamPages() { + return spamPages; + } + + public void setSpamPages(RefreshCommentPages spamPages) { + this.spamPages = spamPages; + } + @Override public String toString() { return ReflectionToStringBuilder.toString(this, SIMPLE_STYLE); diff --git a/src/main/java/io/mattw/youtube/commentsuite/refresh/RefreshReplyPages.java b/src/main/java/io/mattw/youtube/commentsuite/refresh/RefreshReplyPages.java deleted file mode 100644 index 371a135..0000000 --- a/src/main/java/io/mattw/youtube/commentsuite/refresh/RefreshReplyPages.java +++ /dev/null @@ -1,53 +0,0 @@ -package io.mattw.youtube.commentsuite.refresh; - -public enum RefreshReplyPages { - NONE("None", 0), - ALL("All", Integer.MAX_VALUE), - PAGES_1(1), - PAGES_2(2), - PAGES_3(3), - PAGES_4(4), - PAGES_5(5), - PAGES_10(10), - PAGES_15(15), - PAGES_20(20), - PAGES_25(25), - PAGES_50(50), - PAGES_75(75), - PAGES_100(100), - PAGES_125(125), - PAGES_150(150), - PAGES_175(175), - PAGES_200(200), - PAGES_250(250), - PAGES_300(300), - PAGES_400(400), - PAGES_500(500) - ; - - private String displayText; - private int pageCount; - - RefreshReplyPages(String displayText, int pageCount) { - this.displayText = displayText; - this.pageCount = pageCount; - } - - RefreshReplyPages(int pageCount) { - this.displayText = String.valueOf(pageCount); - this.pageCount = pageCount; - } - - public String getDisplayText() { - return displayText; - } - - public int getPageCount() { - return pageCount; - } - - public String toString() { - return getDisplayText(); - } - -} diff --git a/src/main/java/io/mattw/youtube/commentsuite/refresh/RefreshStyle.java b/src/main/java/io/mattw/youtube/commentsuite/refresh/RefreshStyle.java index 3d5432e..8b136ba 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/refresh/RefreshStyle.java +++ b/src/main/java/io/mattw/youtube/commentsuite/refresh/RefreshStyle.java @@ -1,12 +1,12 @@ package io.mattw.youtube.commentsuite.refresh; public enum RefreshStyle { - VIDEO_META_ONLY("Video Meta only", RefreshTimeframe.NONE, RefreshCommentPages.NONE, RefreshCommentOrder.TIME, RefreshReplyPages.NONE), - SKIM("Skim", RefreshTimeframe.ALL, RefreshCommentPages.PAGES_1, RefreshCommentOrder.RELEVANCE, RefreshReplyPages.PAGES_1), - LIGHT("Light", RefreshTimeframe.ALL, RefreshCommentPages.PAGES_5, RefreshCommentOrder.RELEVANCE, RefreshReplyPages.PAGES_5), - MODERATE("Moderate", RefreshTimeframe.ALL, RefreshCommentPages.PAGES_25, RefreshCommentOrder.TIME, RefreshReplyPages.PAGES_25), - HEAVY("Heavy", RefreshTimeframe.ALL, RefreshCommentPages.PAGES_50, RefreshCommentOrder.TIME, RefreshReplyPages.PAGES_50), - EVERYTHING("Everything", RefreshTimeframe.ALL, RefreshCommentPages.ALL, RefreshCommentOrder.TIME, RefreshReplyPages.ALL), + VIDEO_META_ONLY("Video Meta only", RefreshTimeframe.NONE, RefreshCommentPages.NONE, RefreshCommentOrder.TIME, RefreshCommentPages.NONE), + SKIM("Skim", RefreshTimeframe.ALL, RefreshCommentPages.PAGES_1, RefreshCommentOrder.RELEVANCE, RefreshCommentPages.PAGES_1), + LIGHT("Light", RefreshTimeframe.ALL, RefreshCommentPages.PAGES_5, RefreshCommentOrder.RELEVANCE, RefreshCommentPages.PAGES_5), + MODERATE("Moderate", RefreshTimeframe.ALL, RefreshCommentPages.PAGES_25, RefreshCommentOrder.TIME, RefreshCommentPages.PAGES_25), + HEAVY("Heavy", RefreshTimeframe.ALL, RefreshCommentPages.PAGES_50, RefreshCommentOrder.TIME, RefreshCommentPages.PAGES_50), + EVERYTHING("Everything", RefreshTimeframe.ALL, RefreshCommentPages.ALL, RefreshCommentOrder.TIME, RefreshCommentPages.ALL), CUSTOM("Custom") ; @@ -14,13 +14,15 @@ public enum RefreshStyle { private RefreshTimeframe timeframe; private RefreshCommentPages commentPages; private RefreshCommentOrder commentOrder; - private RefreshReplyPages replyPages; + private RefreshCommentPages replyPages; + private RefreshCommentPages reviewPages; + private RefreshCommentPages spamPages; RefreshStyle(String displayText) { this.displayText = displayText; } - RefreshStyle(String displayText, RefreshTimeframe timeframe, RefreshCommentPages commentPages, RefreshCommentOrder commentOrder, RefreshReplyPages replyPages) { + RefreshStyle(String displayText, RefreshTimeframe timeframe, RefreshCommentPages commentPages, RefreshCommentOrder commentOrder, RefreshCommentPages replyPages) { this.displayText = displayText; this.timeframe = timeframe; this.commentPages = commentPages; @@ -44,11 +46,19 @@ public RefreshCommentOrder getCommentOrder() { return commentOrder; } - public RefreshReplyPages getReplyPages() { + public RefreshCommentPages getReplyPages() { return replyPages; } - public boolean matches(RefreshTimeframe timeframe, RefreshCommentPages commentPages, RefreshCommentOrder commentOrder, RefreshReplyPages replyPages) { + public RefreshCommentPages getReviewPages() { + return reviewPages; + } + + public RefreshCommentPages getSpamPages() { + return spamPages; + } + + public boolean matches(RefreshTimeframe timeframe, RefreshCommentPages commentPages, RefreshCommentOrder commentOrder, RefreshCommentPages replyPages) { return this.timeframe == timeframe && this.commentPages == commentPages && this.commentOrder == commentOrder && this.replyPages == replyPages; } diff --git a/src/main/java/io/mattw/youtube/commentsuite/refresh/ReplyProducer.java b/src/main/java/io/mattw/youtube/commentsuite/refresh/ReplyProducer.java index 9d9e0ef..774b474 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/refresh/ReplyProducer.java +++ b/src/main/java/io/mattw/youtube/commentsuite/refresh/ReplyProducer.java @@ -3,7 +3,7 @@ import com.google.api.services.youtube.YouTube; import com.google.api.services.youtube.model.Comment; import com.google.api.services.youtube.model.CommentListResponse; -import io.mattw.youtube.commentsuite.FXMLSuite; +import io.mattw.youtube.commentsuite.CommentSuite; import io.mattw.youtube.commentsuite.db.YouTubeChannel; import io.mattw.youtube.commentsuite.db.YouTubeComment; import io.mattw.youtube.commentsuite.util.ExecutorGroup; @@ -32,7 +32,7 @@ public class ReplyProducer extends ConsumerMultiProducer { public ReplyProducer(final RefreshOptions options) { this.options = options; - this.youTube = FXMLSuite.getYouTube(); + this.youTube = CommentSuite.getYouTube(); } @Override @@ -54,11 +54,11 @@ private void produce() { CommentListResponse response; String pageToken = ""; int page = 1; - RefreshReplyPages replyPages = options.getReplyPages(); + RefreshCommentPages replyPages = options.getReplyPages(); do { response = youTube.comments() .list("snippet") - .setKey(FXMLSuite.getYouTubeApiKey()) + .setKey(CommentSuite.getYouTubeApiKey()) .setParentId(tuple.getFirst()) .setPageToken(pageToken) .setMaxResults(100L) diff --git a/src/main/java/io/mattw/youtube/commentsuite/refresh/UniqueVideoIdProducer.java b/src/main/java/io/mattw/youtube/commentsuite/refresh/UniqueVideoIdProducer.java index f7d3fed..231a923 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/refresh/UniqueVideoIdProducer.java +++ b/src/main/java/io/mattw/youtube/commentsuite/refresh/UniqueVideoIdProducer.java @@ -1,6 +1,6 @@ package io.mattw.youtube.commentsuite.refresh; -import io.mattw.youtube.commentsuite.FXMLSuite; +import io.mattw.youtube.commentsuite.CommentSuite; import io.mattw.youtube.commentsuite.db.CommentDatabase; import io.mattw.youtube.commentsuite.util.ExecutorGroup; import org.apache.logging.log4j.LogManager; @@ -22,7 +22,7 @@ public class UniqueVideoIdProducer extends ConsumerMultiProducer { private long newVideos; public UniqueVideoIdProducer() { - this.database = FXMLSuite.getDatabase(); + this.database = CommentSuite.getDatabase(); } @Override diff --git a/src/main/java/io/mattw/youtube/commentsuite/refresh/VideoIdProducer.java b/src/main/java/io/mattw/youtube/commentsuite/refresh/VideoIdProducer.java index e73a68b..158109f 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/refresh/VideoIdProducer.java +++ b/src/main/java/io/mattw/youtube/commentsuite/refresh/VideoIdProducer.java @@ -2,7 +2,7 @@ import com.google.api.services.youtube.YouTube; import com.google.api.services.youtube.model.*; -import io.mattw.youtube.commentsuite.FXMLSuite; +import io.mattw.youtube.commentsuite.CommentSuite; import io.mattw.youtube.commentsuite.db.CommentDatabase; import io.mattw.youtube.commentsuite.db.GroupItem; import io.mattw.youtube.commentsuite.db.GroupItemVideo; @@ -28,8 +28,8 @@ public class VideoIdProducer extends ConsumerMultiProducer { private final CommentDatabase database; public VideoIdProducer() { - this.youTube = FXMLSuite.getYouTube(); - this.database = FXMLSuite.getDatabase(); + this.youTube = CommentSuite.getYouTube(); + this.database = CommentSuite.getDatabase(); } @Override @@ -73,7 +73,7 @@ private void fromChannel(final GroupItem channel) throws IOException, SQLExcepti logger.debug("fromChannel {}", channel); final ChannelListResponse response = youTube.channels() .list("contentDetails") - .setKey(FXMLSuite.getYouTubeApiKey()) + .setKey(CommentSuite.getYouTubeApiKey()) .setId(channel.getId()) .execute(); @@ -102,7 +102,7 @@ private void fromPlaylist(final GroupItem item, final String playlistId) throws do { response = youTube.playlistItems() .list("snippet") - .setKey(FXMLSuite.getYouTubeApiKey()) + .setKey(CommentSuite.getYouTubeApiKey()) .setMaxResults(50L) .setPlaylistId(playlistId) .setPageToken(pageToken) diff --git a/src/main/java/io/mattw/youtube/commentsuite/refresh/VideoProducer.java b/src/main/java/io/mattw/youtube/commentsuite/refresh/VideoProducer.java index 49256ff..00921c3 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/refresh/VideoProducer.java +++ b/src/main/java/io/mattw/youtube/commentsuite/refresh/VideoProducer.java @@ -3,7 +3,7 @@ import com.google.api.services.youtube.YouTube; import com.google.api.services.youtube.model.VideoListResponse; import io.mattw.youtube.commentsuite.ConfigData; -import io.mattw.youtube.commentsuite.FXMLSuite; +import io.mattw.youtube.commentsuite.CommentSuite; import io.mattw.youtube.commentsuite.db.CommentDatabase; import io.mattw.youtube.commentsuite.db.YouTubeObject; import io.mattw.youtube.commentsuite.db.YouTubeVideo; @@ -24,6 +24,9 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; +import static io.mattw.youtube.commentsuite.refresh.ModerationStatus.HELD_FOR_REVIEW; +import static io.mattw.youtube.commentsuite.refresh.ModerationStatus.LIKELY_SPAM; + public class VideoProducer extends ConsumerMultiProducer { private static final Logger logger = LogManager.getLogger(); @@ -40,9 +43,9 @@ public class VideoProducer extends ConsumerMultiProducer { public VideoProducer(RefreshOptions options) { this.options = options; - this.youTube = FXMLSuite.getYouTube(); - this.database = FXMLSuite.getDatabase(); - this.configData = FXMLSuite.getConfig().getDataObject(); + this.youTube = CommentSuite.getYouTube(); + this.database = CommentSuite.getDatabase(); + this.configData = CommentSuite.getConfig().getDataObject(); } @Override @@ -79,7 +82,7 @@ private void queryAndInsert(final List videoIds) throws IOException, SQL final YouTube.Videos.List response = youTube.videos() .list("snippet,statistics") - .setKey(FXMLSuite.getYouTubeApiKey()) + .setKey(CommentSuite.getYouTubeApiKey()) .setMaxResults(50L) .setId(String.join(",", videoIds)); @@ -122,6 +125,16 @@ private void queryAndInsert(final List videoIds) throws IOException, SQL .collect(Collectors.toList()); logger.debug("Video(s) matches a signed in account {}", videosMine.stream().map(YouTubeObject::getId).collect(Collectors.toList())); + + if (configData.isGrabHeldForReview()) { + logger.debug("Sending review?"); + sendCollection(videosMine, YouTubeVideo.class, HELD_FOR_REVIEW.name()); + } + + if (configData.isGrabLikelySpam()) { + logger.debug("Sending spam?"); + sendCollection(videosMine, YouTubeVideo.class, LIKELY_SPAM.name()); + } } @Override diff --git a/src/main/java/io/mattw/youtube/commentsuite/util/BrowserUtil.java b/src/main/java/io/mattw/youtube/commentsuite/util/BrowserUtil.java index ac71e25..78c4f1a 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/util/BrowserUtil.java +++ b/src/main/java/io/mattw/youtube/commentsuite/util/BrowserUtil.java @@ -6,7 +6,6 @@ /** * Attempts to open URLs on the system. * - * @author mattwright324 */ public class BrowserUtil { diff --git a/src/main/java/io/mattw/youtube/commentsuite/util/ClipboardUtil.java b/src/main/java/io/mattw/youtube/commentsuite/util/ClipboardUtil.java index 869f336..21cbe58 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/util/ClipboardUtil.java +++ b/src/main/java/io/mattw/youtube/commentsuite/util/ClipboardUtil.java @@ -9,9 +9,6 @@ import java.util.List; import java.util.stream.Collectors; -/** - * @author mattwright324 - */ public class ClipboardUtil { private Clipboard systemClipboard; diff --git a/src/main/java/io/mattw/youtube/commentsuite/util/DateUtils.java b/src/main/java/io/mattw/youtube/commentsuite/util/DateUtils.java index 3607ace..6ad86e5 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/util/DateUtils.java +++ b/src/main/java/io/mattw/youtube/commentsuite/util/DateUtils.java @@ -12,14 +12,14 @@ public class DateUtils { /** * Uses the current timezone. */ - public static LocalDateTime epochMillisToDateTime(long epochMillis) { + public static LocalDateTime epochMillisToDateTime(final long epochMillis) { return epochMillisToDateTime(epochMillis, ZoneId.systemDefault()); } /** * Pass in custom timeZone */ - public static LocalDateTime epochMillisToDateTime(long epochMillis, ZoneId zoneId) { + public static LocalDateTime epochMillisToDateTime(final long epochMillis, final ZoneId zoneId) { return LocalDateTime.ofInstant(Instant.ofEpochMilli(epochMillis), zoneId); } diff --git a/src/main/java/io/mattw/youtube/commentsuite/util/ElapsedTime.java b/src/main/java/io/mattw/youtube/commentsuite/util/ElapsedTime.java index 9f6bd31..65e46ad 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/util/ElapsedTime.java +++ b/src/main/java/io/mattw/youtube/commentsuite/util/ElapsedTime.java @@ -6,7 +6,6 @@ /** * Gives a duration from a given date and the current time. * - * @author mattwright324 */ public class ElapsedTime { diff --git a/src/main/java/io/mattw/youtube/commentsuite/util/ExecutorGroup.java b/src/main/java/io/mattw/youtube/commentsuite/util/ExecutorGroup.java index 834debe..77bc424 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/util/ExecutorGroup.java +++ b/src/main/java/io/mattw/youtube/commentsuite/util/ExecutorGroup.java @@ -10,7 +10,6 @@ /** * Structure to make managing a group of threads running the same task in an ExecutorService easier. * - * @author mattwright324 */ public class ExecutorGroup { diff --git a/src/main/java/io/mattw/youtube/commentsuite/util/FXUtils.java b/src/main/java/io/mattw/youtube/commentsuite/util/FXUtils.java index 2ca8391..28151e0 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/util/FXUtils.java +++ b/src/main/java/io/mattw/youtube/commentsuite/util/FXUtils.java @@ -18,11 +18,11 @@ public class FXUtils { private static final Map size = new HashMap<>(); private static final Map padding = new HashMap<>(); - public static void registerToSize(TextField field, int fontSize) { + public static void registerToSize(final TextField field, final int fontSize) { size.put(field, new Font(fontSize)); } - public static void registerToPadding(TextField field, double paddingSize) { + public static void registerToPadding(final TextField field, final double paddingSize) { padding.put(field, paddingSize); } @@ -32,7 +32,7 @@ public static void registerToPadding(TextField field, double paddingSize) { * @param field TextField element * @link https://stackoverflow.com/a/25643696/2650847 */ - public static void adjustTextFieldWidthByContent(TextField field) { + public static void adjustTextFieldWidthByContent(final TextField field) { runLater(() -> { final Text text = new Text(field.getText()); text.setFont(size.getOrDefault(field, field.getFont())); diff --git a/src/main/java/io/mattw/youtube/commentsuite/util/IpApiProvider.java b/src/main/java/io/mattw/youtube/commentsuite/util/IpApiProvider.java index f992bab..27d6d36 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/util/IpApiProvider.java +++ b/src/main/java/io/mattw/youtube/commentsuite/util/IpApiProvider.java @@ -1,8 +1,5 @@ package io.mattw.youtube.commentsuite.util; -/** - * @author mattwright324 - */ public class IpApiProvider implements LocationProvider { @Override diff --git a/src/main/java/io/mattw/youtube/commentsuite/util/Location.java b/src/main/java/io/mattw/youtube/commentsuite/util/Location.java index 052af94..35c7fe0 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/util/Location.java +++ b/src/main/java/io/mattw/youtube/commentsuite/util/Location.java @@ -13,7 +13,6 @@ /** * @param LocationProvider * @param return data object - * @author mattwright324 */ public class Location { diff --git a/src/main/java/io/mattw/youtube/commentsuite/util/LocationProvider.java b/src/main/java/io/mattw/youtube/commentsuite/util/LocationProvider.java index f84e492..b6f0ac4 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/util/LocationProvider.java +++ b/src/main/java/io/mattw/youtube/commentsuite/util/LocationProvider.java @@ -1,8 +1,5 @@ package io.mattw.youtube.commentsuite.util; -/** - * @author mattwright324 - */ public interface LocationProvider { String getRequestUrl(String ipv4); } diff --git a/src/main/java/io/mattw/youtube/commentsuite/util/StringTuple.java b/src/main/java/io/mattw/youtube/commentsuite/util/StringTuple.java index d535c10..3edbc91 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/util/StringTuple.java +++ b/src/main/java/io/mattw/youtube/commentsuite/util/StringTuple.java @@ -5,7 +5,7 @@ public class StringTuple { private final String first; private final String second; - public StringTuple(String first, String second) { + public StringTuple(final String first, final String second) { this.first = first; this.second = second; } diff --git a/src/main/java/io/mattw/youtube/commentsuite/util/Threads.java b/src/main/java/io/mattw/youtube/commentsuite/util/Threads.java new file mode 100644 index 0000000..043920b --- /dev/null +++ b/src/main/java/io/mattw/youtube/commentsuite/util/Threads.java @@ -0,0 +1,12 @@ +package io.mattw.youtube.commentsuite.util; + +public class Threads { + + public static void awaitMillis(final long millis) { + try { + Thread.sleep(millis); + } catch (InterruptedException ignored) { + } + } + +} diff --git a/src/main/java/io/mattw/youtube/commentsuite/util/UTF8UrlEncoder.java b/src/main/java/io/mattw/youtube/commentsuite/util/UTF8UrlEncoder.java index ac79c04..1bb1db8 100644 --- a/src/main/java/io/mattw/youtube/commentsuite/util/UTF8UrlEncoder.java +++ b/src/main/java/io/mattw/youtube/commentsuite/util/UTF8UrlEncoder.java @@ -8,7 +8,6 @@ *

* That exception should never ever be thrown, if it does, the world must be ending. * - * @author mattwright324 */ public class UTF8UrlEncoder { diff --git a/src/main/resources/META-INF/MANIFEST.MF b/src/main/resources/META-INF/MANIFEST.MF index 500f84d..0c026e7 100644 --- a/src/main/resources/META-INF/MANIFEST.MF +++ b/src/main/resources/META-INF/MANIFEST.MF @@ -10,5 +10,5 @@ Class-Path: lib/checker-qual-3.5.0.jar lib/commons-codec-1.10.jar lib/co .13.1.jar lib/jsr305-3.0.2.jar lib/listenablefuture-9999.0-empty-to-avo id-conflict-with-guava.jar lib/log4j-api-2.14.0.jar lib/log4j-core-2.14 .0.jar lib/sqlite-jdbc-3.34.0.jar -Main-Class: io.mattw.youtube.commentsuite.FXMLSuite +Main-Class: io.mattw.youtube.commentsuite.CommentSuite diff --git a/src/main/resources/SuiteStyles.css b/src/main/resources/SuiteStyles.css index 82edfd9..3b9b35f 100644 --- a/src/main/resources/SuiteStyles.css +++ b/src/main/resources/SuiteStyles.css @@ -161,15 +161,27 @@ -fx-cursor: hand; -fx-background-color: linear-gradient(to left, lightgray, transparent); } + .reply { -fx-background-color: linear-gradient(to right, rgba(127,255,127,0.2), transparent); } +.heldForReview { + -fx-background-color: linear-gradient(to right, rgba(255, 255, 127, 0.2), transparent); +} +.likelySpam { + -fx-background-color: linear-gradient(to right, rgba(127,127,127,0.2), transparent); +} .commentDate { -fx-padding: 0 8 0 8; -fx-background-color: derive(rgba(255,255,255,0.5), 70%); -fx-background-radius: 5; } +.tag { + -fx-padding: 0 8 0 8; + -fx-background-color: derive(rgba(255,255,255,0.5), 70%); + -fx-background-radius: 5; + } .commentLikes { -fx-padding: 0 8 0 8; diff --git a/src/main/resources/io/mattw/youtube/commentsuite/db/sql/ddl_create_db.sql b/src/main/resources/io/mattw/youtube/commentsuite/db/sql/ddl_create_db.sql index 135c009..15ce1f5 100644 --- a/src/main/resources/io/mattw/youtube/commentsuite/db/sql/ddl_create_db.sql +++ b/src/main/resources/io/mattw/youtube/commentsuite/db/sql/ddl_create_db.sql @@ -1,73 +1,106 @@ -CREATE TABLE IF NOT EXISTS gitem_type ( - type_id INTEGER PRIMARY KEY, - nameProperty STRING +CREATE TABLE IF NOT EXISTS gitem_type +( + type_id INTEGER PRIMARY KEY, + nameProperty STRING ); -INSERT OR IGNORE INTO gitem_type VALUES - (0, 'video'), - (1, 'channel'), - (2, 'playlist'); -CREATE TABLE IF NOT EXISTS gitem_list ( - gitem_id STRING PRIMARY KEY, - type_id INTEGER, - title STRING, - channel_title STRING, - published DATE, - last_checked DATE, - thumb_url STRING, - FOREIGN KEY(type_id) REFERENCES gitem_type(type_id) + +INSERT OR IGNORE INTO gitem_type +VALUES (0, 'video'), + (1, 'channel'), + (2, 'playlist'); + +CREATE TABLE IF NOT EXISTS gitem_list +( + gitem_id STRING PRIMARY KEY, + type_id INTEGER, + title STRING, + channel_title STRING, + published DATE, + last_checked DATE, + thumb_url STRING, + FOREIGN KEY (type_id) REFERENCES gitem_type (type_id) ); -CREATE TABLE IF NOT EXISTS groups ( - group_id STRING PRIMARY KEY, - group_name STRING UNIQUE + +CREATE TABLE IF NOT EXISTS groups +( + group_id STRING PRIMARY KEY, + group_name STRING UNIQUE ); -CREATE TABLE IF NOT EXISTS group_gitem ( - group_id STRING, - gitem_id STRING, - PRIMARY KEY(group_id, gitem_id), - FOREIGN KEY(group_id) REFERENCES groups(group_id), - FOREIGN KEY(gitem_id) REFERENCES gitem_list(gitem_id) + +CREATE TABLE IF NOT EXISTS group_gitem +( + group_id STRING, + gitem_id STRING, + PRIMARY KEY (group_id, gitem_id), + FOREIGN KEY (group_id) REFERENCES groups (group_id), + FOREIGN KEY (gitem_id) REFERENCES gitem_list (gitem_id) ); -CREATE TABLE IF NOT EXISTS gitem_video ( - gitem_id STRING, - video_id STRING, - PRIMARY KEY(gitem_id, video_id), - FOREIGN KEY(gitem_id) REFERENCES gitem_list(gitem_id), - FOREIGN KEY(video_id) REFERENCES videos(video_id) + +CREATE TABLE IF NOT EXISTS gitem_video +( + gitem_id STRING, + video_id STRING, + PRIMARY KEY (gitem_id, video_id), + FOREIGN KEY (gitem_id) REFERENCES gitem_list (gitem_id), + FOREIGN KEY (video_id) REFERENCES videos (video_id) ); -CREATE TABLE IF NOT EXISTS videos ( - video_id STRING PRIMARY KEY, - channel_id STRING, - grab_date INTEGER, - publish_date INTEGER, - video_title STRING, - total_comments INTEGER, - total_views INTEGER, - total_likes INTGEGER, - total_dislikes INTEGER, - video_desc STRING, - thumb_url STRING, - http_code int, - FOREIGN KEY(channel_id) REFERENCES channels(channel_id) + +CREATE TABLE IF NOT EXISTS videos +( + video_id STRING PRIMARY KEY, + channel_id STRING, + grab_date INTEGER, + publish_date INTEGER, + video_title STRING, + total_comments INTEGER, + total_views INTEGER, + total_likes INTGEGER, + total_dislikes INTEGER, + video_desc STRING, + thumb_url STRING, + http_code int, + FOREIGN KEY (channel_id) REFERENCES channels (channel_id) ); CREATE UNIQUE INDEX IF NOT EXISTS idx_videos_id ON videos (video_id); -CREATE TABLE IF NOT EXISTS comments ( - comment_id STRING PRIMARY KEY, - channel_id STRING, - video_id STRING, - comment_date INTEGER, - comment_likes INTEGER, - reply_count INTEGER, - is_reply BOOLEAN, - parent_id STRING, - comment_text TEXT, - FOREIGN KEY(channel_id) REFERENCES channels(channel_id), - FOREIGN KEY(video_id) REFERENCES videos(video_id) + +CREATE TABLE IF NOT EXISTS comments +( + comment_id STRING PRIMARY KEY, + channel_id STRING, + video_id STRING, + comment_date INTEGER, + comment_likes INTEGER, + reply_count INTEGER, + is_reply BOOLEAN, + parent_id STRING, + comment_text TEXT, + FOREIGN KEY (channel_id) REFERENCES channels (channel_id), + FOREIGN KEY (video_id) REFERENCES videos (video_id) ); CREATE UNIQUE INDEX IF NOT EXISTS idx_comments_id ON comments (comment_id); -CREATE TABLE IF NOT EXISTS channels ( - channel_id STRING PRIMARY KEY, - channel_name STRING, - channel_profile_url STRING, - download_profile BOOLEAN + +CREATE TABLE IF NOT EXISTS comments_moderated +( + comment_id STRING PRIMARY KEY, + channel_id STRING, + video_id STRING, + comment_date INTEGER, + comment_likes INTEGER, + reply_count INTEGER, + is_reply BOOLEAN, + parent_id STRING, + comment_text TEXT, + moderation_status STRING, + FOREIGN KEY (channel_id) REFERENCES channels (channel_id), + FOREIGN KEY (video_id) REFERENCES videos (video_id) +); +CREATE UNIQUE INDEX IF NOT EXISTS idx_comments_id ON comments (comment_id); + +CREATE TABLE IF NOT EXISTS channels +( + channel_id STRING PRIMARY KEY, + channel_name STRING, + channel_profile_url STRING, + download_profile BOOLEAN ); CREATE UNIQUE INDEX IF NOT EXISTS idx_channels_id ON channels (channel_id); \ No newline at end of file diff --git a/src/main/resources/io/mattw/youtube/commentsuite/db/sql/dml_clean_db.sql b/src/main/resources/io/mattw/youtube/commentsuite/db/sql/dml_clean_db.sql index a958024..a310cf4 100644 --- a/src/main/resources/io/mattw/youtube/commentsuite/db/sql/dml_clean_db.sql +++ b/src/main/resources/io/mattw/youtube/commentsuite/db/sql/dml_clean_db.sql @@ -3,7 +3,9 @@ DELETE FROM gitem_list WHERE gitem_id NOT IN (SELECT DISTINCT gitem_id FROM grou DELETE FROM gitem_video WHERE gitem_id NOT IN (SELECT DISTINCT gitem_id FROM gitem_list); DELETE FROM videos WHERE video_id NOT IN (SELECT DISTINCT video_id FROM gitem_video); DELETE FROM comments WHERE video_id NOT IN (SELECT DISTINCT video_id FROM videos); +DELETE FROM comments_moderated WHERE video_id NOT IN (SELECT DISTINCT video_id FROM videos); DELETE FROM channels WHERE channel_id NOT IN ( SELECT DISTINCT channel_id FROM videos UNION SELECT channel_id FROM comments + UNION SELECT channel_id FROM comments_moderated ); \ No newline at end of file diff --git a/src/main/resources/io/mattw/youtube/commentsuite/db/sql/dml_delete_comment.sql b/src/main/resources/io/mattw/youtube/commentsuite/db/sql/dml_delete_comment.sql new file mode 100644 index 0000000..ad358a2 --- /dev/null +++ b/src/main/resources/io/mattw/youtube/commentsuite/db/sql/dml_delete_comment.sql @@ -0,0 +1 @@ +DELETE FROM comments WHERE comment_id = ? \ No newline at end of file diff --git a/src/main/resources/io/mattw/youtube/commentsuite/db/sql/dml_delete_moderated_comment.sql b/src/main/resources/io/mattw/youtube/commentsuite/db/sql/dml_delete_moderated_comment.sql new file mode 100644 index 0000000..0df7748 --- /dev/null +++ b/src/main/resources/io/mattw/youtube/commentsuite/db/sql/dml_delete_moderated_comment.sql @@ -0,0 +1 @@ +DELETE FROM comments_moderated WHERE comment_id = ? \ No newline at end of file diff --git a/src/main/resources/io/mattw/youtube/commentsuite/db/sql/dml_insert_ignore_moderated_comments.sql b/src/main/resources/io/mattw/youtube/commentsuite/db/sql/dml_insert_ignore_moderated_comments.sql new file mode 100644 index 0000000..9048aec --- /dev/null +++ b/src/main/resources/io/mattw/youtube/commentsuite/db/sql/dml_insert_ignore_moderated_comments.sql @@ -0,0 +1,3 @@ +INSERT OR IGNORE INTO comments_moderated ( + comment_id, channel_id, video_id, comment_date, comment_text, comment_likes, reply_count, is_reply, parent_id, moderation_status +) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) \ No newline at end of file diff --git a/src/main/resources/io/mattw/youtube/commentsuite/db/sql/dql_does_moderated_comment_exist.sql b/src/main/resources/io/mattw/youtube/commentsuite/db/sql/dql_does_moderated_comment_exist.sql new file mode 100644 index 0000000..186f83b --- /dev/null +++ b/src/main/resources/io/mattw/youtube/commentsuite/db/sql/dql_does_moderated_comment_exist.sql @@ -0,0 +1 @@ +SELECT 1 FROM comments_moderated WHERE comment_id = ?; \ No newline at end of file diff --git a/src/main/resources/io/mattw/youtube/commentsuite/db/sql/dql_get_comment_tree.sql b/src/main/resources/io/mattw/youtube/commentsuite/db/sql/dql_get_comment_tree.sql index cd933fa..bdb3be6 100644 --- a/src/main/resources/io/mattw/youtube/commentsuite/db/sql/dql_get_comment_tree.sql +++ b/src/main/resources/io/mattw/youtube/commentsuite/db/sql/dql_get_comment_tree.sql @@ -1,4 +1,10 @@ -SELECT * FROM comments +SELECT *, null as moderation_status FROM comments JOIN channels USING (channel_id) WHERE comment_id = ? OR parent_id = ? -ORDER BY is_reply, comment_date; \ No newline at end of file + +UNION ALL + +SELECT * FROM comments_moderated +JOIN channels USING (channel_id) +WHERE ? = true AND (comment_id = ? OR parent_id = ?) +ORDER BY is_reply, comment_date \ No newline at end of file diff --git a/src/main/resources/io/mattw/youtube/commentsuite/db/sql/dql_get_moderated_comment_stats.sql b/src/main/resources/io/mattw/youtube/commentsuite/db/sql/dql_get_moderated_comment_stats.sql new file mode 100644 index 0000000..5fe36d2 --- /dev/null +++ b/src/main/resources/io/mattw/youtube/commentsuite/db/sql/dql_get_moderated_comment_stats.sql @@ -0,0 +1,7 @@ +SELECT COUNT(comment_id) AS total_comments +FROM comments_moderated +WHERE video_id IN ( + SELECT video_id FROM gitem_video + JOIN group_gitem USING (gitem_id) + WHERE group_id = ? +); \ No newline at end of file diff --git a/src/main/resources/io/mattw/youtube/commentsuite/fxml/MGMVRefreshModal.fxml b/src/main/resources/io/mattw/youtube/commentsuite/fxml/MGMVRefreshModal.fxml index 363c48f..76e1a32 100644 --- a/src/main/resources/io/mattw/youtube/commentsuite/fxml/MGMVRefreshModal.fxml +++ b/src/main/resources/io/mattw/youtube/commentsuite/fxml/MGMVRefreshModal.fxml @@ -31,6 +31,16 @@ + + + + + + + + + + - - + diff --git a/src/main/resources/io/mattw/youtube/commentsuite/fxml/ManageGroupsManager.fxml b/src/main/resources/io/mattw/youtube/commentsuite/fxml/ManageGroupsManager.fxml index 6589eed..9cce95d 100644 --- a/src/main/resources/io/mattw/youtube/commentsuite/fxml/ManageGroupsManager.fxml +++ b/src/main/resources/io/mattw/youtube/commentsuite/fxml/ManageGroupsManager.fxml @@ -54,30 +54,11 @@ - + - + - - + - + - diff --git a/src/main/resources/io/mattw/youtube/commentsuite/fxml/SearchCommentsListItem.fxml b/src/main/resources/io/mattw/youtube/commentsuite/fxml/SearchCommentsListItem.fxml index 3ab7648..7772844 100644 --- a/src/main/resources/io/mattw/youtube/commentsuite/fxml/SearchCommentsListItem.fxml +++ b/src/main/resources/io/mattw/youtube/commentsuite/fxml/SearchCommentsListItem.fxml @@ -28,6 +28,9 @@ + + + \ No newline at end of file diff --git a/src/main/resources/io/mattw/youtube/commentsuite/fxml/Settings.fxml b/src/main/resources/io/mattw/youtube/commentsuite/fxml/Settings.fxml index cf4396c..0425254 100644 --- a/src/main/resources/io/mattw/youtube/commentsuite/fxml/Settings.fxml +++ b/src/main/resources/io/mattw/youtube/commentsuite/fxml/Settings.fxml @@ -60,6 +60,8 @@