diff --git a/airsonic-main/src/main/java/org/airsonic/player/controller/MusicFolderSettingsController.java b/airsonic-main/src/main/java/org/airsonic/player/controller/MusicFolderSettingsController.java index 2dade807..c8f5fafa 100644 --- a/airsonic-main/src/main/java/org/airsonic/player/controller/MusicFolderSettingsController.java +++ b/airsonic-main/src/main/java/org/airsonic/player/controller/MusicFolderSettingsController.java @@ -26,6 +26,7 @@ import org.airsonic.player.dao.MediaFileDao; import org.airsonic.player.domain.MusicFolder; import org.airsonic.player.service.MediaScannerService; import org.airsonic.player.service.SettingsService; +import org.airsonic.player.service.search.IndexManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -63,6 +64,8 @@ public class MusicFolderSettingsController { private AlbumDao albumDao; @Autowired private MediaFileDao mediaFileDao; + @Autowired + private IndexManager indexManager; @GetMapping protected String displayForm() throws Exception { @@ -98,6 +101,14 @@ public class MusicFolderSettingsController { private void expunge() { + + // to be before dao#expunge + LOG.debug("Cleaning search index..."); + indexManager.startIndexing(); + indexManager.expunge(); + indexManager.stopIndexing(); + LOG.debug("Search index cleanup complete."); + LOG.debug("Cleaning database..."); LOG.debug("Deleting non-present artists..."); artistDao.expunge(); diff --git a/airsonic-main/src/main/java/org/airsonic/player/dao/AlbumDao.java b/airsonic-main/src/main/java/org/airsonic/player/dao/AlbumDao.java index 1eefb278..a4b9ba6c 100644 --- a/airsonic-main/src/main/java/org/airsonic/player/dao/AlbumDao.java +++ b/airsonic-main/src/main/java/org/airsonic/player/dao/AlbumDao.java @@ -343,6 +343,10 @@ public class AlbumDao extends AbstractDao { } } + public List getExpungeCandidates() { + return queryForInts("select id from album where not present"); + } + public void expunge() { int minId = queryForInt("select min(id) from album where not present", 0); int maxId = queryForInt("select max(id) from album where not present", 0); diff --git a/airsonic-main/src/main/java/org/airsonic/player/dao/ArtistDao.java b/airsonic-main/src/main/java/org/airsonic/player/dao/ArtistDao.java index 8d3f94ae..8b607062 100644 --- a/airsonic-main/src/main/java/org/airsonic/player/dao/ArtistDao.java +++ b/airsonic-main/src/main/java/org/airsonic/player/dao/ArtistDao.java @@ -169,6 +169,10 @@ public class ArtistDao extends AbstractDao { } } + public List getExpungeCandidates() { + return queryForInts("select id from artist where not present"); + } + public void expunge() { int minId = queryForInt("select min(id) from artist where not present", 0); int maxId = queryForInt("select max(id) from artist where not present", 0); diff --git a/airsonic-main/src/main/java/org/airsonic/player/dao/MediaFileDao.java b/airsonic-main/src/main/java/org/airsonic/player/dao/MediaFileDao.java index fd9f97a9..ed9a91e3 100644 --- a/airsonic-main/src/main/java/org/airsonic/player/dao/MediaFileDao.java +++ b/airsonic-main/src/main/java/org/airsonic/player/dao/MediaFileDao.java @@ -661,6 +661,22 @@ public class MediaFileDao extends AbstractDao { } } + public List getArtistExpungeCandidates() { + return queryForInts("select id from media_file where media_file.type = ? and not present", + MediaFile.MediaType.DIRECTORY.name()); + } + + public List getAlbumExpungeCandidates() { + return queryForInts("select id from media_file where media_file.type = ? and not present", + MediaFile.MediaType.ALBUM.name()); + } + + public List getSongExpungeCandidates() { + return queryForInts("select id from media_file where media_file.type in (?,?,?,?) and not present", + MediaFile.MediaType.MUSIC.name(), MediaFile.MediaType.PODCAST.name(), + MediaFile.MediaType.AUDIOBOOK.name(), MediaFile.MediaType.VIDEO.name()); + } + public void expunge() { int minId = queryForInt("select min(id) from media_file where not present", 0); int maxId = queryForInt("select max(id) from media_file where not present", 0); diff --git a/airsonic-main/src/main/java/org/airsonic/player/service/search/DocumentFactory.java b/airsonic-main/src/main/java/org/airsonic/player/service/search/DocumentFactory.java index 1f548b69..6c2923e3 100644 --- a/airsonic-main/src/main/java/org/airsonic/player/service/search/DocumentFactory.java +++ b/airsonic-main/src/main/java/org/airsonic/player/service/search/DocumentFactory.java @@ -127,16 +127,20 @@ public class DocumentFactory { doc.add(new SortedDocValuesField(fieldName, new BytesRef(value))); }; + public final Term createPrimarykey(Integer id) { + return new Term(FieldNames.ID, Integer.toString(id)); + }; + public final Term createPrimarykey(Album album) { - return new Term(FieldNames.ID, Integer.toString(album.getId())); + return createPrimarykey(album.getId()); }; public final Term createPrimarykey(Artist artist) { - return new Term(FieldNames.ID, Integer.toString(artist.getId())); + return createPrimarykey(artist.getId()); }; public final Term createPrimarykey(MediaFile mediaFile) { - return new Term(FieldNames.ID, Integer.toString(mediaFile.getId())); + return createPrimarykey(mediaFile.getId()); }; /** diff --git a/airsonic-main/src/main/java/org/airsonic/player/service/search/IndexManager.java b/airsonic-main/src/main/java/org/airsonic/player/service/search/IndexManager.java index de5290fb..89048073 100644 --- a/airsonic-main/src/main/java/org/airsonic/player/service/search/IndexManager.java +++ b/airsonic-main/src/main/java/org/airsonic/player/service/search/IndexManager.java @@ -20,6 +20,9 @@ package org.airsonic.player.service.search; +import org.airsonic.player.dao.AlbumDao; +import org.airsonic.player.dao.ArtistDao; +import org.airsonic.player.dao.MediaFileDao; import org.airsonic.player.domain.Album; import org.airsonic.player.domain.Artist; import org.airsonic.player.domain.MediaFile; @@ -99,6 +102,15 @@ public class IndexManager { @Autowired private DocumentFactory documentFactory; + @Autowired + private MediaFileDao mediaFileDao; + + @Autowired + private ArtistDao artistDao; + + @Autowired + private AlbumDao albumDao; + private Map searchers = new HashMap<>(); private Map writers = new HashMap<>(); @@ -163,6 +175,55 @@ public class IndexManager { return new IndexWriter(FSDirectory.open(indexDirectory.toPath()), config); } + public void expunge() { + + Term[] primarykeys = mediaFileDao.getArtistExpungeCandidates().stream() + .map(m -> documentFactory.createPrimarykey(m)) + .toArray(i -> new Term[i]); + try { + writers.get(IndexType.ARTIST).deleteDocuments(primarykeys); + } catch (IOException e) { + LOG.error("Failed to delete artist doc.", e); + } + + primarykeys = mediaFileDao.getAlbumExpungeCandidates().stream() + .map(m -> documentFactory.createPrimarykey(m)) + .toArray(i -> new Term[i]); + try { + writers.get(IndexType.ALBUM).deleteDocuments(primarykeys); + } catch (IOException e) { + LOG.error("Failed to delete album doc.", e); + } + + primarykeys = mediaFileDao.getSongExpungeCandidates().stream() + .map(m -> documentFactory.createPrimarykey(m)) + .toArray(i -> new Term[i]); + try { + writers.get(IndexType.SONG).deleteDocuments(primarykeys); + } catch (IOException e) { + LOG.error("Failed to delete song doc.", e); + } + + primarykeys = artistDao.getExpungeCandidates().stream() + .map(m -> documentFactory.createPrimarykey(m)) + .toArray(i -> new Term[i]); + try { + writers.get(IndexType.ARTIST_ID3).deleteDocuments(primarykeys); + } catch (IOException e) { + LOG.error("Failed to delete artistId3 doc.", e); + } + + primarykeys = albumDao.getExpungeCandidates().stream() + .map(m -> documentFactory.createPrimarykey(m)) + .toArray(i -> new Term[i]); + try { + writers.get(IndexType.ALBUM_ID3).deleteDocuments(primarykeys); + } catch (IOException e) { + LOG.error("Failed to delete albumId3 doc.", e); + } + + } + /** * Close Writer of all indexes and update SearcherManager. * Called at the end of the Scan flow. diff --git a/airsonic-main/src/test/java/org/airsonic/player/service/search/IndexManagerTestCase.java b/airsonic-main/src/test/java/org/airsonic/player/service/search/IndexManagerTestCase.java new file mode 100644 index 00000000..96744cb1 --- /dev/null +++ b/airsonic-main/src/test/java/org/airsonic/player/service/search/IndexManagerTestCase.java @@ -0,0 +1,201 @@ +/* + This file is part of Airsonic. + + Airsonic is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Airsonic is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with Airsonic. If not, see . + + Copyright 2016 (C) Airsonic Authors + Based upon Subsonic, Copyright 2009 (C) Sindre Mehus + */ +package org.airsonic.player.service.search; + +import org.airsonic.player.dao.AlbumDao; +import org.airsonic.player.dao.ArtistDao; +import org.airsonic.player.dao.MediaFileDao; +import org.airsonic.player.domain.MusicFolder; +import org.airsonic.player.domain.SearchCriteria; +import org.airsonic.player.domain.SearchResult; +import org.airsonic.player.service.SearchService; +import org.airsonic.player.service.search.IndexManager; +import org.airsonic.player.service.search.IndexType; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.ResourceLoader; + +import java.io.File; +import java.util.Date; +import java.util.List; +import java.util.ArrayList; + +import static org.junit.Assert.assertEquals; +import static org.springframework.util.ObjectUtils.isEmpty; + +public class IndexManagerTestCase extends AbstractAirsonicHomeTest { + + private List musicFolders; + + @Autowired + private SearchService searchService; + + @Autowired + private IndexManager indexManager; + + @Override + public List getMusicFolders() { + if (isEmpty(musicFolders)) { + musicFolders = new ArrayList<>(); + File musicDir = new File(resolveBaseMediaPath.apply("Music")); + musicFolders.add(new MusicFolder(1, musicDir, "Music", true, new Date())); + } + return musicFolders; + } + + @Before + public void setup() throws Exception { + populateDatabaseOnlyOnce(); + } + + @Autowired + private MediaFileDao mediaFileDao; + + @Autowired + private ArtistDao artistDao; + + @Autowired + private AlbumDao albumDao; + + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Autowired + ResourceLoader resourceLoader; + + @Test + public void testExpunge() throws InterruptedException { + + SearchCriteria criteria = new SearchCriteria(); + criteria.setOffset(0); + criteria.setCount(Integer.MAX_VALUE); + criteria.setQuery("_DIR_ Ravel"); + + SearchCriteria criteriaSong = new SearchCriteria(); + criteriaSong.setOffset(0); + criteriaSong.setCount(Integer.MAX_VALUE); + criteriaSong.setQuery("Gaspard"); + + SearchCriteria criteriaAlbumId3 = new SearchCriteria(); + criteriaAlbumId3.setOffset(0); + criteriaAlbumId3.setCount(Integer.MAX_VALUE); + criteriaAlbumId3.setQuery("Complete Piano Works"); + + /* Delete DB record. */ + + // artist + SearchResult result = searchService.search(criteria, musicFolders, IndexType.ARTIST); + assertEquals(2, result.getMediaFiles().size()); + assertEquals("_DIR_ Ravel", result.getMediaFiles().get(0).getName()); + assertEquals("_DIR_ Sixteen Horsepower", result.getMediaFiles().get(1).getName()); + + List candidates = mediaFileDao.getArtistExpungeCandidates(); + assertEquals(0, candidates.size()); + + result.getMediaFiles().forEach(a -> mediaFileDao.deleteMediaFile(a.getPath())); + + candidates = mediaFileDao.getArtistExpungeCandidates(); + assertEquals(2, candidates.size()); + + // album + result = searchService.search(criteria, musicFolders, IndexType.ALBUM); + assertEquals(2, result.getMediaFiles().size()); + assertEquals("_DIR_ Ravel - Complete Piano Works", result.getMediaFiles().get(0).getName()); + assertEquals("_DIR_ Ravel - Chamber Music With Voice", result.getMediaFiles().get(1).getName()); + + candidates = mediaFileDao.getAlbumExpungeCandidates(); + assertEquals(0, candidates.size()); + + result.getMediaFiles().forEach(a -> mediaFileDao.deleteMediaFile(a.getPath())); + + candidates = mediaFileDao.getAlbumExpungeCandidates(); + assertEquals(2, candidates.size()); + + // song + result = searchService.search(criteriaSong, musicFolders, IndexType.SONG); + assertEquals(2, result.getMediaFiles().size()); + assertEquals("01 - Gaspard de la Nuit - i. Ondine", result.getMediaFiles().get(0).getName()); + assertEquals("02 - Gaspard de la Nuit - ii. Le Gibet", result.getMediaFiles().get(1).getName()); + + candidates = mediaFileDao.getSongExpungeCandidates(); + assertEquals(0, candidates.size()); + + result.getMediaFiles().forEach(a -> mediaFileDao.deleteMediaFile(a.getPath())); + + candidates = mediaFileDao.getSongExpungeCandidates(); + assertEquals(2, candidates.size()); + + // artistid3 + result = searchService.search(criteria, musicFolders, IndexType.ARTIST_ID3); + assertEquals(1, result.getArtists().size()); + assertEquals("_DIR_ Ravel", result.getArtists().get(0).getName()); + + candidates = artistDao.getExpungeCandidates(); + assertEquals(0, candidates.size()); + + artistDao.markNonPresent(new Date()); + + candidates = artistDao.getExpungeCandidates(); + assertEquals(4, candidates.size()); + + // albumId3 + result = searchService.search(criteriaAlbumId3, musicFolders, IndexType.ALBUM_ID3); + assertEquals(1, result.getAlbums().size()); + assertEquals("Complete Piano Works", result.getAlbums().get(0).getName()); + + candidates = albumDao.getExpungeCandidates(); + assertEquals(0, candidates.size()); + + albumDao.markNonPresent(new Date()); + + candidates = albumDao.getExpungeCandidates(); + assertEquals(4, candidates.size()); + + /* Does not scan, only expunges the index. */ + indexManager.startIndexing(); + indexManager.expunge(); + indexManager.stopIndexing(); + + /* + * Subsequent search results. + * Results can also be confirmed with Luke. + */ + + result = searchService.search(criteria, musicFolders, IndexType.ARTIST); + assertEquals(0, result.getMediaFiles().size()); + + result = searchService.search(criteria, musicFolders, IndexType.ALBUM); + assertEquals(0, result.getMediaFiles().size()); + + result = searchService.search(criteriaSong, musicFolders, IndexType.SONG); + assertEquals(0, result.getMediaFiles().size()); + + result = searchService.search(criteria, musicFolders, IndexType.ARTIST_ID3); + assertEquals(0, result.getArtists().size()); + + result = searchService.search(criteriaAlbumId3, musicFolders, IndexType.ALBUM_ID3); + assertEquals(0, result.getAlbums().size()); + + } + +} \ No newline at end of file