Skip to content

Commit 759a915

Browse files
trinidztrinidz
authored
Merge pull request #1 from trinidz/avgposttime
Avgposttime
2 parents 0ade936 + 23ae159 commit 759a915

File tree

6 files changed

+100
-52
lines changed

6 files changed

+100
-52
lines changed

‎event.go

Lines changed: 16 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@ import (
1616

1717
const KIND_BOOKMARKS int = 10003 //NIP-51
1818

19-
// gets notes from the local db
2019
func getLocalEvents(localFilter nostr.Filter) ([]*nostr.Event, error) {
2120
ctx := context.TODO()
2221

@@ -191,7 +190,7 @@ func deleteRemoteFollow(pubkeyHex string) nostr.Tags {
191190
return nil
192191
}
193192

194-
// TRUE if a rsslay feedUrl already exists in bookmark event
193+
// TRUE if feed exists in bookmark event
195194
func feedExists(pubkeyHex, privKeyHex, feedUrl string) (bool, error) {
196195
var bookMarkTags nostr.Tags
197196

@@ -220,7 +219,6 @@ func feedExists(pubkeyHex, privKeyHex, feedUrl string) (bool, error) {
220219
return false, nil
221220
}
222221

223-
// add a feed entity to the bookmark event
224222
func addEntityToBookmarkEvent(entitiesToAdd []Entity) error {
225223
if len(entitiesToAdd) == 0 {
226224
return nil
@@ -273,10 +271,10 @@ func addEntityToBookmarkEvent(entitiesToAdd []Entity) error {
273271
return nil
274272
}
275273

276-
// update an existing feed entity 'last' property in the bookmark event
277-
func updateEntityInBookmarkEvent(pubKeyHex string, lastUpdate int64) error {
274+
// update an existing feed entity time properties
275+
func updateEntityTimesInBookmarkEvent(updatedEntity Entity) error {
278276
var bookMarkTags nostr.Tags
279-
var rsslayEntity Entity
277+
var rssnotesEntity Entity
280278

281279
var bookmarkFilter nostr.Filter = nostr.Filter{
282280
Kinds: []int{KIND_BOOKMARKS},
@@ -292,18 +290,20 @@ func updateEntityInBookmarkEvent(pubKeyHex string, lastUpdate int64) error {
292290
if len(bookMarkEvts) > 0 {
293291
bookMarkTags = bookMarkEvts[0].Tags.GetAll([]string{s.RsslayTagKey})
294292
for i, tag := range bookMarkTags {
295-
if strings.Contains(tag.Value(), pubKeyHex) {
296-
if err := json.Unmarshal([]byte(tag.Value()), &rsslayEntity); err != nil {
293+
if strings.Contains(tag.Value(), updatedEntity.PubKey) {
294+
if err := json.Unmarshal([]byte(tag.Value()), &rssnotesEntity); err != nil {
297295
log.Printf("[ERROR] %s", err)
298296
}
299297

300298
copy(bookMarkTags[i:], bookMarkTags[i+1:])
301299
bookMarkTags[len(bookMarkTags)-1] = nostr.Tag{}
302300
bookMarkTags = bookMarkTags[:len(bookMarkTags)-1]
303301

304-
rsslayEntity.LastUpdate = lastUpdate
302+
rssnotesEntity.LastPostTime = updatedEntity.LastPostTime
303+
rssnotesEntity.LastCheckedTime = updatedEntity.LastCheckedTime
304+
rssnotesEntity.AvgPostTime = updatedEntity.AvgPostTime
305305

306-
jsonentArr, err := json.Marshal(rsslayEntity)
306+
jsonentArr, err := json.Marshal(rssnotesEntity)
307307
if err != nil {
308308
log.Printf("[ERROR] %s", err)
309309
}
@@ -313,7 +313,7 @@ func updateEntityInBookmarkEvent(pubKeyHex string, lastUpdate int64) error {
313313
evt := nostr.Event{
314314
CreatedAt: nostr.Now(),
315315
Kind: KIND_BOOKMARKS,
316-
Content: "{rsslay, pubkey, privkey, url, last_update}",
316+
Content: "",
317317
Tags: bookMarkTags,
318318
}
319319

@@ -326,7 +326,7 @@ func updateEntityInBookmarkEvent(pubKeyHex string, lastUpdate int64) error {
326326
store(context.TODO(), &evt)
327327
}
328328

329-
log.Printf("[DEBUG] entity %s last update %d in event ID %s", rsslayEntity.URL, rsslayEntity.LastUpdate, evt.ID)
329+
log.Printf("[DEBUG] entity %s last post time %d in event ID %s", rssnotesEntity.URL, rssnotesEntity.LastPostTime, evt.ID)
330330
break
331331
}
332332
}
@@ -337,7 +337,6 @@ func updateEntityInBookmarkEvent(pubKeyHex string, lastUpdate int64) error {
337337
return nil
338338
}
339339

340-
// delete a feed entity from a local bookmark event
341340
func deleteEntityInBookmarkEvent(pubKeyORfeedUrl string) error {
342341
var bookMarkTags nostr.Tags
343342
var rsslayEntity Entity
@@ -412,7 +411,6 @@ func deleteEntityInBookmarkEvent(pubKeyORfeedUrl string) error {
412411
return nil
413412
}
414413

415-
// return all saved FeedURL entries
416414
func getSavedEntries() ([]GUIEntry, error) {
417415

418416
var bookMarkTags nostr.Tags
@@ -441,10 +439,10 @@ func getSavedEntries() ([]GUIEntry, error) {
441439
npub, _ := nip19.EncodePublicKey(rsslayEntity.PubKey)
442440
localEntries = append(localEntries, GUIEntry{
443441
BookmarkEntity: Entity{
444-
PubKey: rsslayEntity.PubKey,
445-
URL: rsslayEntity.URL,
446-
ImageURL: rsslayEntity.ImageURL,
447-
LastUpdate: rsslayEntity.LastUpdate},
442+
PubKey: rsslayEntity.PubKey,
443+
URL: rsslayEntity.URL,
444+
ImageURL: rsslayEntity.ImageURL,
445+
LastPostTime: rsslayEntity.LastPostTime},
448446
NPubKey: npub,
449447
})
450448
}
@@ -454,7 +452,6 @@ func getSavedEntries() ([]GUIEntry, error) {
454452
return localEntries, nil
455453
}
456454

457-
// find the saved Entity with the given pubkey
458455
func getSavedEntity(pubkeyHex string) (Entity, error) {
459456
var bookMarkTags nostr.Tags
460457
var rsslayEntity Entity
@@ -487,7 +484,6 @@ func getSavedEntity(pubkeyHex string) (Entity, error) {
487484
return Entity{}, nil
488485
}
489486

490-
// return all feedURL Entities
491487
func getSavedEntities() ([]Entity, error) {
492488
var bookMarkTags nostr.Tags
493489
var rsslayEntity Entity
@@ -518,7 +514,6 @@ func getSavedEntities() ([]Entity, error) {
518514
return entities, nil
519515
}
520516

521-
// delete event from relay db
522517
func deleteLocalEvents(filter nostr.Filter) error {
523518
ctx := context.TODO()
524519

‎feed.go

Lines changed: 30 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -348,15 +348,21 @@ func GetConverterRules() []md.Rule {
348348
}
349349
}
350350

351-
func updateAllFeeds() {
352-
var latestCreatedAt int64
353-
//metrics.ListeningFiltersOps.Inc()
351+
func checkAllFeeds() {
354352
currentEntities, err := getSavedEntities()
355353
if err != nil {
356354
log.Print("[ERROR] could not retrieve entities")
357355
return
358356
}
359357
for _, currentEntity := range currentEntities {
358+
if !TimetoUpdateFeed(currentEntity) {
359+
//log.Printf("[DEBUG] not time to update %s. Time since last check: %d Avg post time: %d", currentEntity.URL, time.Now().Unix()-currentEntity.LastCheckedTime, currentEntity.AvgPostTime)
360+
continue
361+
}
362+
363+
lastPostTime := int64(0)
364+
allPostTimes := make([]int64, 0)
365+
360366
parsedFeed, entity := parseFeedForPubkey(currentEntity.PubKey, s.DeleteFailingFeeds)
361367
if parsedFeed == nil {
362368
return
@@ -369,7 +375,7 @@ func updateAllFeeds() {
369375
for _, item := range parsedFeed.Items {
370376
defaultCreatedAt := time.Unix(time.Now().Unix(), 0)
371377
evt := feedItemToNote(currentEntity.PubKey, item, parsedFeed, defaultCreatedAt, entity.URL, s.MaxContentLength)
372-
if entity.LastUpdate < evt.CreatedAt.Time().Unix() {
378+
if entity.LastPostTime < evt.CreatedAt.Time().Unix() {
373379
if err := evt.Sign(entity.PrivateKey); err != nil {
374380
log.Printf("[ERROR] %s", err)
375381
return
@@ -381,25 +387,30 @@ func updateAllFeeds() {
381387
for _, store := range relay.StoreEvent {
382388
store(context.TODO(), &evt)
383389
}
384-
} else {
385-
log.Printf("[DEBUG] event id %s created at %d older than last update %d", evt.ID, evt.CreatedAt.Time().Unix(), entity.LastUpdate)
386390
}
387-
if evt.CreatedAt.Time().Unix() > latestCreatedAt {
388-
latestCreatedAt = evt.CreatedAt.Time().Unix()
391+
392+
if evt.CreatedAt.Time().Unix() > lastPostTime {
393+
lastPostTime = evt.CreatedAt.Time().Unix()
389394
}
395+
396+
allPostTimes = append(allPostTimes, evt.CreatedAt.Time().Unix())
390397
}
391398

392-
if err := updateEntityInBookmarkEvent(entity.PubKey, latestCreatedAt); err != nil {
399+
if err := updateEntityTimesInBookmarkEvent(Entity{
400+
PubKey: entity.PubKey,
401+
LastPostTime: lastPostTime,
402+
LastCheckedTime: time.Now().Unix(),
403+
AvgPostTime: CalcAvgPostTime(allPostTimes),
404+
}); err != nil {
393405
log.Printf("[ERROR] feed entity %s not updated", entity.URL)
394406
}
395-
396-
latestCreatedAt = 0
397407
}
398408
}
399409

400-
// init feed by creating kind 1's from rss feed
401-
func initFeed(pubkey string, privkey string, feedURL string, parsedFeed *gofeed.Feed) int64 {
402-
var latestCreatedAt int64
410+
func initFeed(pubkey string, privkey string, feedURL string, parsedFeed *gofeed.Feed) (int64, []int64) {
411+
var lastPostTime int64
412+
postTimes := make([]int64, 0)
413+
403414
for _, item := range parsedFeed.Items {
404415
defaultCreatedAt := time.Unix(time.Now().Unix(), 0)
405416
evt := feedItemToNote(pubkey, item, parsedFeed, defaultCreatedAt, feedURL, s.MaxContentLength)
@@ -415,10 +426,12 @@ func initFeed(pubkey string, privkey string, feedURL string, parsedFeed *gofeed.
415426
store(context.TODO(), &evt)
416427
}
417428

418-
if evt.CreatedAt.Time().Unix() > latestCreatedAt {
419-
latestCreatedAt = evt.CreatedAt.Time().Unix()
429+
if evt.CreatedAt.Time().Unix() > lastPostTime {
430+
lastPostTime = evt.CreatedAt.Time().Unix()
420431
}
432+
433+
postTimes = append(postTimes, evt.CreatedAt.Time().Unix())
421434
}
422435

423-
return latestCreatedAt
436+
return lastPostTime, postTimes
424437
}

‎handlers.go

Lines changed: 18 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -166,14 +166,16 @@ func createFeed(r *http.Request, secret *string) *GUIEntry {
166166
publishNostrEventCh <- metadataEvent
167167
}
168168

169-
latestCreatedAt := initFeed(publicKey, sk, feedUrl, parsedFeed)
169+
lastPostTime, allPostTimes := initFeed(publicKey, sk, feedUrl, parsedFeed)
170170

171171
if err := addEntityToBookmarkEvent([]Entity{
172172
{PubKey: publicKey,
173-
PrivateKey: sk,
174-
URL: feedUrl,
175-
ImageURL: guientry.BookmarkEntity.ImageURL,
176-
LastUpdate: latestCreatedAt}}); err != nil {
173+
PrivateKey: sk,
174+
URL: feedUrl,
175+
ImageURL: guientry.BookmarkEntity.ImageURL,
176+
LastPostTime: lastPostTime,
177+
LastCheckedTime: time.Now().Unix(),
178+
AvgPostTime: CalcAvgPostTime(allPostTimes)}}); err != nil {
177179
log.Printf("[ERROR] feed entity %s not added to bookmark", feedUrl)
178180
}
179181

@@ -383,8 +385,16 @@ func importFeeds(opmlUrls []opml.Outline, secret *string) []*GUIEntry {
383385
localImageURL = parsedFeed.Image.URL
384386
}
385387

386-
latestCreatedAt := initFeed(publicKey, sk, feedUrl, parsedFeed)
387-
bookmarkEntities = append(bookmarkEntities, Entity{PubKey: publicKey, PrivateKey: sk, URL: feedUrl, ImageURL: localImageURL, LastUpdate: latestCreatedAt})
388+
lastPostTime, allPostTimes := initFeed(publicKey, sk, feedUrl, parsedFeed)
389+
bookmarkEntities = append(bookmarkEntities, Entity{
390+
PubKey: publicKey,
391+
PrivateKey: sk,
392+
URL: feedUrl,
393+
ImageURL: localImageURL,
394+
LastPostTime: lastPostTime,
395+
LastCheckedTime: time.Now().Unix(),
396+
AvgPostTime: CalcAvgPostTime(allPostTimes),
397+
})
388398

389399
importedEntries = append(importedEntries, &guiEntry)
390400
importProgressCh <- ImportProgressStruct{entryIndex: urlIndex, totalEntries: len(opmlUrls)}
@@ -586,7 +596,7 @@ func updateRssNotesState() {
586596
blastEvent(&nostrEvent)
587597
}()
588598
case <-tickerUpdateFeeds.C:
589-
updateAllFeeds()
599+
checkAllFeeds()
590600
case <-tickerDeleteOldNotes.C:
591601
deleteOldEvents()
592602
case <-quitChannel:

‎helpers.go

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"os"
1010
"path"
1111
"slices"
12+
"sort"
1213
"strings"
1314
"time"
1415
)
@@ -82,3 +83,27 @@ func GetRelayListFromFile(filePath string) []string {
8283
}
8384
return relayList
8485
}
86+
87+
func CalcAvgPostTime(feedPostTimes []int64) int64 {
88+
if len(feedPostTimes) < s.MinPostPeriodSamples {
89+
return int64(s.MaxAvgPostPeriodHrs * 60 * 60)
90+
}
91+
92+
sort.SliceStable(feedPostTimes, func(i, j int) bool {
93+
return feedPostTimes[i] > feedPostTimes[j]
94+
})
95+
96+
avgposttimesecs := (feedPostTimes[0] - feedPostTimes[len(feedPostTimes)-1]) / int64(len(feedPostTimes))
97+
98+
if avgposttimesecs < int64(s.MinAvgPostPeriodMins*60) {
99+
return int64(s.MinAvgPostPeriodMins * 60)
100+
} else if avgposttimesecs > int64(s.MaxAvgPostPeriodHrs*60*60) {
101+
return int64(s.MaxAvgPostPeriodHrs * 60 * 60)
102+
}
103+
104+
return avgposttimesecs
105+
}
106+
107+
func TimetoUpdateFeed(rssfeed Entity) bool {
108+
return time.Now().Unix()-rssfeed.LastCheckedTime >= rssfeed.AvgPostTime
109+
}

‎main.go

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ type Settings struct {
3131
RelayIcon string `envconfig:"RELAY_ICON" default:"https://cdn.jsdelivr.net/gh/walkxcode/dashboard-icons/png/commafeed.png"`
3232
RandomSecret string `envconfig:"RANDOM_SECRET" required:"true"`
3333
OwnerPubkey string `envconfig:"OWNER_PUBKEY"`
34-
Version string `envconfig:"VERSION" default:"0.0.7"`
34+
Version string `envconfig:"VERSION" default:"0.0.8"`
3535

3636
LogLevel string `envconfig:"LOG_LEVEL" default:"WARN"`
3737
Port string `envconfig:"PORT" default:"3334"`
@@ -50,6 +50,9 @@ type Settings struct {
5050
FeedItemsRefreshMinutes int `envconfig:"FEED_ITEMS_REFRESH_MINUTES" default:"30"`
5151
FeedMetadataRefreshDays int `envconfig:"METADATA_REFRESH_DAYS" default:"7"`
5252
MaxNoteAgeDays int `envconfig:"MAX_NOTE_AGE_DAYS" default:"0"`
53+
MaxAvgPostPeriodHrs int64 `envconfig:"MAX_AVG_POST_PERIOD_HRS" default:"4"`
54+
MinAvgPostPeriodMins int64 `envconfig:"MIN_AVG_POST_PERIOD_MINS" default:"10"`
55+
MinPostPeriodSamples int `envconfig:"MIN_POST_PERIOD_SAMPLES" default:"5"`
5356
}
5457

5558
var (

‎models.go

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -26,11 +26,13 @@ type ImportProgressStruct struct {
2626
}
2727

2828
type Entity struct {
29-
PubKey string
30-
PrivateKey string
31-
URL string
32-
ImageURL string
33-
LastUpdate int64
29+
PubKey string
30+
PrivateKey string
31+
URL string
32+
ImageURL string
33+
LastPostTime int64
34+
AvgPostTime int64
35+
LastCheckedTime int64
3436
}
3537

3638
type GUIEntry struct {

0 commit comments

Comments
 (0)